summaryrefslogtreecommitdiffstats
path: root/taskcluster/gecko_taskgraph/test
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
commit36d22d82aa202bb199967e9512281e9a53db42c9 (patch)
tree105e8c98ddea1c1e4784a60a5a6410fa416be2de /taskcluster/gecko_taskgraph/test
parentInitial commit. (diff)
downloadfirefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.tar.xz
firefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.zip
Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
-rw-r--r--taskcluster/gecko_taskgraph/test/__init__.py0
-rw-r--r--taskcluster/gecko_taskgraph/test/automationrelevance.json358
-rw-r--r--taskcluster/gecko_taskgraph/test/conftest.py220
-rw-r--r--taskcluster/gecko_taskgraph/test/docs/kinds.rst12
-rw-r--r--taskcluster/gecko_taskgraph/test/docs/parameters.rst14
-rw-r--r--taskcluster/gecko_taskgraph/test/python.ini23
-rw-r--r--taskcluster/gecko_taskgraph/test/test_actions_util.py179
-rw-r--r--taskcluster/gecko_taskgraph/test/test_decision.py176
-rw-r--r--taskcluster/gecko_taskgraph/test/test_files_changed.py90
-rw-r--r--taskcluster/gecko_taskgraph/test/test_main.py67
-rw-r--r--taskcluster/gecko_taskgraph/test/test_morph.py108
-rw-r--r--taskcluster/gecko_taskgraph/test/test_optimize_strategies.py551
-rw-r--r--taskcluster/gecko_taskgraph/test/test_target_tasks.py371
-rw-r--r--taskcluster/gecko_taskgraph/test/test_taskcluster_yml.py145
-rw-r--r--taskcluster/gecko_taskgraph/test/test_transforms_job.py150
-rw-r--r--taskcluster/gecko_taskgraph/test/test_transforms_test.py328
-rw-r--r--taskcluster/gecko_taskgraph/test/test_try_option_syntax.py430
-rw-r--r--taskcluster/gecko_taskgraph/test/test_util_attributes.py99
-rw-r--r--taskcluster/gecko_taskgraph/test/test_util_backstop.py155
-rw-r--r--taskcluster/gecko_taskgraph/test/test_util_bugbug.py57
-rw-r--r--taskcluster/gecko_taskgraph/test/test_util_chunking.py403
-rw-r--r--taskcluster/gecko_taskgraph/test/test_util_docker.py255
-rw-r--r--taskcluster/gecko_taskgraph/test/test_util_partials.py128
-rw-r--r--taskcluster/gecko_taskgraph/test/test_util_runnable_jobs.py76
-rw-r--r--taskcluster/gecko_taskgraph/test/test_util_templates.py79
-rw-r--r--taskcluster/gecko_taskgraph/test/test_util_verify.py149
26 files changed, 4623 insertions, 0 deletions
diff --git a/taskcluster/gecko_taskgraph/test/__init__.py b/taskcluster/gecko_taskgraph/test/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/__init__.py
diff --git a/taskcluster/gecko_taskgraph/test/automationrelevance.json b/taskcluster/gecko_taskgraph/test/automationrelevance.json
new file mode 100644
index 0000000000..3bdfa9ed9e
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/automationrelevance.json
@@ -0,0 +1,358 @@
+{
+ "changesets": [
+ {
+ "author": "James Long <longster@gmail.com>",
+ "backsoutnodes": [],
+ "bugs": [
+ {
+ "no": "1300866",
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300866"
+ }
+ ],
+ "date": [1473196655.0, 14400],
+ "desc": "Bug 1300866 - expose devtools require to new debugger r=jlast,bgrins",
+ "extra": {
+ "branch": "default"
+ },
+ "files": ["devtools/client/debugger/index.html"],
+ "node": "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "parents": ["37c9349b4e8167a61b08b7e119c21ea177b98942"],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "pushdate": [1473261248, 0],
+ "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+ "pushid": 30664,
+ "pushnodes": [
+ "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+ "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+ "99c542fa43a72ee863c813b5624048d1b443549b",
+ "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+ "541c9086c0f27fba60beecc9bc94543103895c86",
+ "041a925171e431bf51fb50193ab19d156088c89a",
+ "a14f88a9af7a59e677478694bafd9375ac53683e"
+ ],
+ "pushuser": "cbook@mozilla.com",
+ "rev": 312890,
+ "reviewers": [
+ {
+ "name": "jlast",
+ "revset": "reviewer(jlast)"
+ },
+ {
+ "name": "bgrins",
+ "revset": "reviewer(bgrins)"
+ }
+ ],
+ "treeherderrepo": "mozilla-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+ },
+ {
+ "author": "Wes Kocher <wkocher@mozilla.com>",
+ "backsoutnodes": [],
+ "bugs": [],
+ "date": [1473208638.0, 25200],
+ "desc": "Merge m-c to fx-team, a=merge",
+ "extra": {
+ "branch": "default"
+ },
+ "files": ["taskcluster/scripts/builder/build-l10n.sh"],
+ "node": "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+ "parents": [
+ "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "91c2b9d5c1354ca79e5b174591dbb03b32b15bbf"
+ ],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "pushdate": [1473261248, 0],
+ "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+ "pushid": 30664,
+ "pushnodes": [
+ "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+ "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+ "99c542fa43a72ee863c813b5624048d1b443549b",
+ "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+ "541c9086c0f27fba60beecc9bc94543103895c86",
+ "041a925171e431bf51fb50193ab19d156088c89a",
+ "a14f88a9af7a59e677478694bafd9375ac53683e"
+ ],
+ "pushuser": "cbook@mozilla.com",
+ "rev": 312891,
+ "reviewers": [
+ {
+ "name": "merge",
+ "revset": "reviewer(merge)"
+ }
+ ],
+ "treeherderrepo": "mozilla-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+ },
+ {
+ "author": "Towkir Ahmed <towkir17@gmail.com>",
+ "backsoutnodes": [],
+ "bugs": [
+ {
+ "no": "1296648",
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1296648"
+ }
+ ],
+ "date": [1472957580.0, 14400],
+ "desc": "Bug 1296648 - Fix direction of .ruleview-expander.theme-twisty in RTL locales. r=ntim",
+ "extra": {
+ "branch": "default"
+ },
+ "files": ["devtools/client/themes/rules.css"],
+ "node": "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+ "parents": ["73a6a267a50a0e1c41e689b265ad3eebe43d7ac6"],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "pushdate": [1473261248, 0],
+ "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+ "pushid": 30664,
+ "pushnodes": [
+ "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+ "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+ "99c542fa43a72ee863c813b5624048d1b443549b",
+ "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+ "541c9086c0f27fba60beecc9bc94543103895c86",
+ "041a925171e431bf51fb50193ab19d156088c89a",
+ "a14f88a9af7a59e677478694bafd9375ac53683e"
+ ],
+ "pushuser": "cbook@mozilla.com",
+ "rev": 312892,
+ "reviewers": [
+ {
+ "name": "ntim",
+ "revset": "reviewer(ntim)"
+ }
+ ],
+ "treeherderrepo": "mozilla-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+ },
+ {
+ "author": "Oriol <oriol-bugzilla@hotmail.com>",
+ "backsoutnodes": [],
+ "bugs": [
+ {
+ "no": "1300336",
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300336"
+ }
+ ],
+ "date": [1472921160.0, 14400],
+ "desc": "Bug 1300336 - Allow pseudo-arrays to have a length property. r=fitzgen",
+ "extra": {
+ "branch": "default"
+ },
+ "files": [
+ "devtools/client/webconsole/test/browser_webconsole_output_06.js",
+ "devtools/server/actors/object.js"
+ ],
+ "node": "99c542fa43a72ee863c813b5624048d1b443549b",
+ "parents": ["16a1a91f9269ab95dd83eb29dc5d0227665f7d94"],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "pushdate": [1473261248, 0],
+ "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+ "pushid": 30664,
+ "pushnodes": [
+ "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+ "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+ "99c542fa43a72ee863c813b5624048d1b443549b",
+ "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+ "541c9086c0f27fba60beecc9bc94543103895c86",
+ "041a925171e431bf51fb50193ab19d156088c89a",
+ "a14f88a9af7a59e677478694bafd9375ac53683e"
+ ],
+ "pushuser": "cbook@mozilla.com",
+ "rev": 312893,
+ "reviewers": [
+ {
+ "name": "fitzgen",
+ "revset": "reviewer(fitzgen)"
+ }
+ ],
+ "treeherderrepo": "mozilla-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+ },
+ {
+ "author": "Ruturaj Vartak <ruturaj@gmail.com>",
+ "backsoutnodes": [],
+ "bugs": [
+ {
+ "no": "1295010",
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010"
+ }
+ ],
+ "date": [1472854020.0, -7200],
+ "desc": "Bug 1295010 - Don't move the eyedropper to the out of browser window by keyboard navigation. r=pbro\n\nMozReview-Commit-ID: vBwmSxVNXK",
+ "extra": {
+ "amend_source": "6885024ef00cfa33d73c59dc03c48ebcda9ccbdd",
+ "branch": "default",
+ "histedit_source": "c43167f0a7cbe9f4c733b15da726e5150a9529ba",
+ "rebase_source": "b74df421630fc46dab6b6cc026bf3e0ae6b4a651"
+ },
+ "files": [
+ "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js",
+ "devtools/client/inspector/test/head.js",
+ "devtools/server/actors/highlighters/eye-dropper.js"
+ ],
+ "node": "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+ "parents": ["99c542fa43a72ee863c813b5624048d1b443549b"],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "pushdate": [1473261248, 0],
+ "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+ "pushid": 30664,
+ "pushnodes": [
+ "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+ "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+ "99c542fa43a72ee863c813b5624048d1b443549b",
+ "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+ "541c9086c0f27fba60beecc9bc94543103895c86",
+ "041a925171e431bf51fb50193ab19d156088c89a",
+ "a14f88a9af7a59e677478694bafd9375ac53683e"
+ ],
+ "pushuser": "cbook@mozilla.com",
+ "rev": 312894,
+ "reviewers": [
+ {
+ "name": "pbro",
+ "revset": "reviewer(pbro)"
+ }
+ ],
+ "treeherderrepo": "mozilla-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+ },
+ {
+ "author": "Matteo Ferretti <mferretti@mozilla.com>",
+ "backsoutnodes": [],
+ "bugs": [
+ {
+ "no": "1299154",
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1299154"
+ }
+ ],
+ "date": [1472629906.0, -7200],
+ "desc": "Bug 1299154 - added Set/GetOverrideDPPX to restorefromHistory; r=mstange\n\nMozReview-Commit-ID: AsyAcG3Igbn\n",
+ "extra": {
+ "branch": "default",
+ "committer": "Matteo Ferretti <mferretti@mozilla.com> 1473236511 -7200"
+ },
+ "files": [
+ "docshell/base/nsDocShell.cpp",
+ "dom/tests/mochitest/general/test_contentViewer_overrideDPPX.html"
+ ],
+ "node": "541c9086c0f27fba60beecc9bc94543103895c86",
+ "parents": ["a6b6a93eb41a05e310a11f0172f01ba9b21d3eac"],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "pushdate": [1473261248, 0],
+ "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+ "pushid": 30664,
+ "pushnodes": [
+ "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+ "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+ "99c542fa43a72ee863c813b5624048d1b443549b",
+ "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+ "541c9086c0f27fba60beecc9bc94543103895c86",
+ "041a925171e431bf51fb50193ab19d156088c89a",
+ "a14f88a9af7a59e677478694bafd9375ac53683e"
+ ],
+ "pushuser": "cbook@mozilla.com",
+ "rev": 312895,
+ "reviewers": [
+ {
+ "name": "mstange",
+ "revset": "reviewer(mstange)"
+ }
+ ],
+ "treeherderrepo": "mozilla-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+ },
+ {
+ "author": "Patrick Brosset <pbrosset@mozilla.com>",
+ "backsoutnodes": [],
+ "bugs": [
+ {
+ "no": "1295010",
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010"
+ }
+ ],
+ "date": [1473239449.0, -7200],
+ "desc": "Bug 1295010 - Removed testActor from highlighterHelper in inspector tests; r=me\n\nMozReview-Commit-ID: GMksl81iGcp",
+ "extra": {
+ "branch": "default"
+ },
+ "files": [
+ "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js",
+ "devtools/client/inspector/test/head.js"
+ ],
+ "node": "041a925171e431bf51fb50193ab19d156088c89a",
+ "parents": ["541c9086c0f27fba60beecc9bc94543103895c86"],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "pushdate": [1473261248, 0],
+ "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+ "pushid": 30664,
+ "pushnodes": [
+ "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+ "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+ "99c542fa43a72ee863c813b5624048d1b443549b",
+ "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+ "541c9086c0f27fba60beecc9bc94543103895c86",
+ "041a925171e431bf51fb50193ab19d156088c89a",
+ "a14f88a9af7a59e677478694bafd9375ac53683e"
+ ],
+ "pushuser": "cbook@mozilla.com",
+ "rev": 312896,
+ "reviewers": [
+ {
+ "name": "me",
+ "revset": "reviewer(me)"
+ }
+ ],
+ "treeherderrepo": "mozilla-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+ },
+ {
+ "author": "Carsten \"Tomcat\" Book <cbook@mozilla.com>",
+ "backsoutnodes": [],
+ "bugs": [],
+ "date": [1473261233.0, -7200],
+ "desc": "merge fx-team to mozilla-central a=merge",
+ "extra": {
+ "branch": "default"
+ },
+ "files": [],
+ "node": "a14f88a9af7a59e677478694bafd9375ac53683e",
+ "parents": [
+ "3d0b41fdd93bd8233745eadb4e0358e385bf2cb9",
+ "041a925171e431bf51fb50193ab19d156088c89a"
+ ],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "pushdate": [1473261248, 0],
+ "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+ "pushid": 30664,
+ "pushnodes": [
+ "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+ "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+ "99c542fa43a72ee863c813b5624048d1b443549b",
+ "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+ "541c9086c0f27fba60beecc9bc94543103895c86",
+ "041a925171e431bf51fb50193ab19d156088c89a",
+ "a14f88a9af7a59e677478694bafd9375ac53683e"
+ ],
+ "pushuser": "cbook@mozilla.com",
+ "rev": 312897,
+ "reviewers": [
+ {
+ "name": "merge",
+ "revset": "reviewer(merge)"
+ }
+ ],
+ "treeherderrepo": "mozilla-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+ }
+ ],
+ "visible": true
+}
diff --git a/taskcluster/gecko_taskgraph/test/conftest.py b/taskcluster/gecko_taskgraph/test/conftest.py
new file mode 100644
index 0000000000..09231c5ab2
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/conftest.py
@@ -0,0 +1,220 @@
+# Any copyright is dedicated to the public domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+import os
+
+import pytest
+from mach.logging import LoggingManager
+from responses import RequestsMock
+from taskgraph import generator as generator_mod
+from taskgraph import target_tasks as target_tasks_mod
+from taskgraph.config import GraphConfig, load_graph_config
+from taskgraph.generator import Kind, TaskGraphGenerator
+from taskgraph.optimize import base as optimize_mod
+from taskgraph.optimize.base import OptimizationStrategy
+from taskgraph.parameters import Parameters
+
+from gecko_taskgraph import GECKO
+from gecko_taskgraph.actions import render_actions_json
+from gecko_taskgraph.util.templates import merge
+
+
+@pytest.fixture
+def responses():
+ with RequestsMock() as rsps:
+ yield rsps
+
+
+@pytest.fixture(scope="session", autouse=True)
+def patch_prefherder(request):
+ from _pytest.monkeypatch import MonkeyPatch
+
+ m = MonkeyPatch()
+ m.setattr(
+ "gecko_taskgraph.util.bugbug._write_perfherder_data",
+ lambda lower_is_better: None,
+ )
+ yield
+ m.undo()
+
+
+@pytest.fixture(scope="session", autouse=True)
+def enable_logging():
+ """Ensure logs from gecko_taskgraph are displayed when a test fails."""
+ lm = LoggingManager()
+ lm.add_terminal_logging()
+
+
+@pytest.fixture(scope="session")
+def graph_config():
+ return load_graph_config(os.path.join(GECKO, "taskcluster", "ci"))
+
+
+@pytest.fixture(scope="session")
+def actions_json(graph_config):
+ decision_task_id = "abcdef"
+ return render_actions_json(Parameters(strict=False), graph_config, decision_task_id)
+
+
+def fake_loader(kind, path, config, parameters, loaded_tasks):
+ for i in range(3):
+ dependencies = {}
+ if i >= 1:
+ dependencies["prev"] = f"{kind}-t-{i - 1}"
+
+ task = {
+ "kind": kind,
+ "label": f"{kind}-t-{i}",
+ "description": f"{kind} task {i}",
+ "attributes": {"_tasknum": str(i)},
+ "task": {
+ "i": i,
+ "metadata": {"name": f"t-{i}"},
+ "deadline": "soon",
+ },
+ "dependencies": dependencies,
+ }
+ if "job-defaults" in config:
+ task = merge(config["job-defaults"], task)
+ yield task
+
+
+class FakeTransform:
+ transforms = []
+ params = {}
+
+ def __init__(self):
+ pass
+
+ @classmethod
+ def get(self, field, default):
+ try:
+ return getattr(self, field)
+ except AttributeError:
+ return default
+
+
+class FakeKind(Kind):
+ def _get_loader(self):
+ return fake_loader
+
+ def load_tasks(self, parameters, loaded_tasks, write_artifacts):
+ FakeKind.loaded_kinds.append(self.name)
+ return super().load_tasks(parameters, loaded_tasks, write_artifacts)
+
+ @staticmethod
+ def create(name, extra_config, graph_config):
+ if name == "fullfake":
+ config = FakeTransform()
+ else:
+ config = {"transforms": []}
+ if extra_config:
+ config.update(extra_config)
+ return FakeKind(name, "/fake", config, graph_config)
+
+
+class WithFakeKind(TaskGraphGenerator):
+ def _load_kinds(self, graph_config, target_kind=None):
+ for kind_name, cfg in self.parameters["_kinds"]:
+ yield FakeKind.create(kind_name, cfg, graph_config)
+
+
+def fake_load_graph_config(root_dir):
+ graph_config = GraphConfig(
+ {"trust-domain": "test-domain", "taskgraph": {}}, root_dir
+ )
+ graph_config.__dict__["register"] = lambda: None
+ return graph_config
+
+
+class FakeParameters(dict):
+ strict = True
+
+ def file_url(self, path, pretty=False):
+ return ""
+
+
+class FakeOptimization(OptimizationStrategy):
+ description = "Fake strategy for testing"
+
+ def __init__(self, mode, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.mode = mode
+
+ def should_remove_task(self, task, params, arg):
+ if self.mode == "always":
+ return True
+ if self.mode == "even":
+ return task.task["i"] % 2 == 0
+ if self.mode == "odd":
+ return task.task["i"] % 2 != 0
+ return False
+
+
+@pytest.fixture
+def maketgg(monkeypatch):
+ def inner(target_tasks=None, kinds=[("_fake", [])], params=None):
+ params = params or {}
+ FakeKind.loaded_kinds = loaded_kinds = []
+ target_tasks = target_tasks or []
+
+ def target_tasks_method(full_task_graph, parameters, graph_config):
+ return target_tasks
+
+ fake_registry = {
+ mode: FakeOptimization(mode) for mode in ("always", "never", "even", "odd")
+ }
+
+ target_tasks_mod._target_task_methods["test_method"] = target_tasks_method
+ monkeypatch.setattr(optimize_mod, "registry", fake_registry)
+
+ parameters = FakeParameters(
+ {
+ "_kinds": kinds,
+ "backstop": False,
+ "enable_always_target": False,
+ "target_tasks_method": "test_method",
+ "test_manifest_loader": "default",
+ "try_mode": None,
+ "try_task_config": {},
+ "tasks_for": "hg-push",
+ "project": "mozilla-central",
+ }
+ )
+ parameters.update(params)
+
+ monkeypatch.setattr(generator_mod, "load_graph_config", fake_load_graph_config)
+
+ tgg = WithFakeKind("/root", parameters)
+ tgg.loaded_kinds = loaded_kinds
+ return tgg
+
+ return inner
+
+
+@pytest.fixture
+def run_transform():
+
+ graph_config = fake_load_graph_config("/root")
+ kind = FakeKind.create("fake", {}, graph_config)
+
+ def inner(xform, tasks):
+ if isinstance(tasks, dict):
+ tasks = [tasks]
+ return xform(kind.config, tasks)
+
+ return inner
+
+
+@pytest.fixture
+def run_full_config_transform():
+
+ graph_config = fake_load_graph_config("/root")
+ kind = FakeKind.create("fullfake", {}, graph_config)
+
+ def inner(xform, tasks):
+ if isinstance(tasks, dict):
+ tasks = [tasks]
+ return xform(kind.config, tasks)
+
+ return inner
diff --git a/taskcluster/gecko_taskgraph/test/docs/kinds.rst b/taskcluster/gecko_taskgraph/test/docs/kinds.rst
new file mode 100644
index 0000000000..fdc16db1e3
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/docs/kinds.rst
@@ -0,0 +1,12 @@
+Task Kinds
+==========
+
+Fake task kind documentation.
+
+newkind
+----------
+Kind found in separate doc dir,
+
+anotherkind
+-----------
+Here's another.
diff --git a/taskcluster/gecko_taskgraph/test/docs/parameters.rst b/taskcluster/gecko_taskgraph/test/docs/parameters.rst
new file mode 100644
index 0000000000..f943f48e69
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/docs/parameters.rst
@@ -0,0 +1,14 @@
+==========
+Parameters
+==========
+
+Fake parameters documentation.
+
+Heading
+-------
+
+``newparameter``
+ A new parameter that could be defined in a project.
+
+``anotherparameter``
+ And here is another one.
diff --git a/taskcluster/gecko_taskgraph/test/python.ini b/taskcluster/gecko_taskgraph/test/python.ini
new file mode 100644
index 0000000000..2dca8fc144
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/python.ini
@@ -0,0 +1,23 @@
+[DEFAULT]
+subsuite = taskgraph
+
+[test_actions_util.py]
+[test_decision.py]
+[test_files_changed.py]
+[test_main.py]
+[test_morph.py]
+[test_optimize_strategies.py]
+[test_target_tasks.py]
+[test_taskcluster_yml.py]
+[test_transforms_job.py]
+[test_transforms_test.py]
+[test_try_option_syntax.py]
+[test_util_attributes.py]
+[test_util_backstop.py]
+[test_util_bugbug.py]
+[test_util_chunking.py]
+[test_util_docker.py]
+[test_util_partials.py]
+[test_util_runnable_jobs.py]
+[test_util_templates.py]
+[test_util_verify.py]
diff --git a/taskcluster/gecko_taskgraph/test/test_actions_util.py b/taskcluster/gecko_taskgraph/test/test_actions_util.py
new file mode 100644
index 0000000000..7c38caea57
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/test_actions_util.py
@@ -0,0 +1,179 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import json
+import unittest
+from pprint import pprint
+from unittest.mock import patch
+
+import pytest
+from mozunit import MockedOpen, main
+from taskgraph import create
+from taskgraph.util import taskcluster
+
+from gecko_taskgraph import actions
+from gecko_taskgraph.actions.util import combine_task_graph_files, relativize_datestamps
+from gecko_taskgraph.decision import read_artifact
+
+TASK_DEF = {
+ "created": "2017-10-10T18:33:03.460Z",
+ # note that this is not an even number of seconds off!
+ "deadline": "2017-10-11T18:33:03.461Z",
+ "dependencies": [],
+ "expires": "2018-10-10T18:33:04.461Z",
+ "payload": {
+ "artifacts": {
+ "public": {
+ "expires": "2018-10-10T18:33:03.463Z",
+ "path": "/builds/worker/artifacts",
+ "type": "directory",
+ },
+ },
+ "maxRunTime": 1800,
+ },
+}
+
+
+@pytest.fixture(scope="module", autouse=True)
+def enable_test_mode():
+ create.testing = True
+ taskcluster.testing = True
+
+
+class TestRelativize(unittest.TestCase):
+ def test_relativize(self):
+ rel = relativize_datestamps(TASK_DEF)
+ import pprint
+
+ pprint.pprint(rel)
+ assert rel["created"] == {"relative-datestamp": "0 seconds"}
+ assert rel["deadline"] == {"relative-datestamp": "86400 seconds"}
+ assert rel["expires"] == {"relative-datestamp": "31536001 seconds"}
+ assert rel["payload"]["artifacts"]["public"]["expires"] == {
+ "relative-datestamp": "31536000 seconds"
+ }
+
+
+class TestCombineTaskGraphFiles(unittest.TestCase):
+ def test_no_suffixes(self):
+ with MockedOpen({}):
+ combine_task_graph_files([])
+ self.assertRaises(Exception, open("artifacts/to-run.json"))
+
+ @patch("gecko_taskgraph.actions.util.rename_artifact")
+ def test_one_suffix(self, rename_artifact):
+ combine_task_graph_files(["0"])
+ rename_artifact.assert_any_call("task-graph-0.json", "task-graph.json")
+ rename_artifact.assert_any_call(
+ "label-to-taskid-0.json", "label-to-taskid.json"
+ )
+ rename_artifact.assert_any_call("to-run-0.json", "to-run.json")
+
+ def test_several_suffixes(self):
+ files = {
+ "artifacts/task-graph-0.json": json.dumps({"taska": {}}),
+ "artifacts/label-to-taskid-0.json": json.dumps({"taska": "TASKA"}),
+ "artifacts/to-run-0.json": json.dumps(["taska"]),
+ "artifacts/task-graph-1.json": json.dumps({"taskb": {}}),
+ "artifacts/label-to-taskid-1.json": json.dumps({"taskb": "TASKB"}),
+ "artifacts/to-run-1.json": json.dumps(["taskb"]),
+ }
+ with MockedOpen(files):
+ combine_task_graph_files(["0", "1"])
+ self.assertEqual(
+ read_artifact("task-graph.json"),
+ {
+ "taska": {},
+ "taskb": {},
+ },
+ )
+ self.assertEqual(
+ read_artifact("label-to-taskid.json"),
+ {
+ "taska": "TASKA",
+ "taskb": "TASKB",
+ },
+ )
+ self.assertEqual(
+ sorted(read_artifact("to-run.json")),
+ [
+ "taska",
+ "taskb",
+ ],
+ )
+
+
+def is_subset(subset, superset):
+ if isinstance(subset, dict):
+ return all(
+ key in superset and is_subset(val, superset[key])
+ for key, val in subset.items()
+ )
+
+ if isinstance(subset, list) or isinstance(subset, set):
+ return all(
+ any(is_subset(subitem, superitem) for superitem in superset)
+ for subitem in subset
+ )
+
+ if isinstance(subset, str):
+ return subset in superset
+
+ # assume that subset is a plain value if none of the above match
+ return subset == superset
+
+
+@pytest.mark.parametrize(
+ "task_def,expected",
+ [
+ pytest.param(
+ {"tags": {"kind": "decision-task"}},
+ {
+ "hookPayload": {
+ "decision": {
+ "action": {"cb_name": "retrigger-decision"},
+ },
+ },
+ },
+ id="retrigger_decision",
+ ),
+ pytest.param(
+ {"tags": {"action": "backfill-task"}},
+ {
+ "hookPayload": {
+ "decision": {
+ "action": {"cb_name": "retrigger-decision"},
+ },
+ },
+ },
+ id="retrigger_backfill",
+ ),
+ ],
+)
+def test_extract_applicable_action(
+ responses, monkeypatch, actions_json, task_def, expected
+):
+ base_url = "https://taskcluster"
+ decision_task_id = "dddd"
+ task_id = "tttt"
+
+ monkeypatch.setenv("TASK_ID", task_id)
+ monkeypatch.setenv("TASKCLUSTER_ROOT_URL", base_url)
+ monkeypatch.setenv("TASKCLUSTER_PROXY_URL", base_url)
+ responses.add(
+ responses.GET,
+ f"{base_url}/api/queue/v1/task/{task_id}",
+ status=200,
+ json=task_def,
+ )
+ action = actions.util._extract_applicable_action(
+ actions_json, "retrigger", decision_task_id, task_id
+ )
+ pprint(action, indent=2)
+ assert is_subset(expected, action)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/taskcluster/gecko_taskgraph/test/test_decision.py b/taskcluster/gecko_taskgraph/test/test_decision.py
new file mode 100644
index 0000000000..81be7e0c44
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/test_decision.py
@@ -0,0 +1,176 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import json
+import os
+import shutil
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import pytest
+from mozunit import MockedOpen, main
+from taskgraph.util.yaml import load_yaml
+
+from gecko_taskgraph import decision
+from gecko_taskgraph.parameters import register_parameters
+
+FAKE_GRAPH_CONFIG = {"product-dir": "browser", "taskgraph": {}}
+
+
+@pytest.fixture(scope="module", autouse=True)
+def register():
+ register_parameters()
+
+
+class TestDecision(unittest.TestCase):
+ def test_write_artifact_json(self):
+ data = [{"some": "data"}]
+ tmpdir = tempfile.mkdtemp()
+ try:
+ decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts")
+ decision.write_artifact("artifact.json", data)
+ with open(os.path.join(decision.ARTIFACTS_DIR, "artifact.json")) as f:
+ self.assertEqual(json.load(f), data)
+ finally:
+ if os.path.exists(tmpdir):
+ shutil.rmtree(tmpdir)
+ decision.ARTIFACTS_DIR = "artifacts"
+
+ def test_write_artifact_yml(self):
+ data = [{"some": "data"}]
+ tmpdir = tempfile.mkdtemp()
+ try:
+ decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts")
+ decision.write_artifact("artifact.yml", data)
+ self.assertEqual(load_yaml(decision.ARTIFACTS_DIR, "artifact.yml"), data)
+ finally:
+ if os.path.exists(tmpdir):
+ shutil.rmtree(tmpdir)
+ decision.ARTIFACTS_DIR = "artifacts"
+
+
+class TestGetDecisionParameters(unittest.TestCase):
+
+ ttc_file = os.path.join(os.getcwd(), "try_task_config.json")
+
+ def setUp(self):
+ self.options = {
+ "base_repository": "https://hg.mozilla.org/mozilla-unified",
+ "head_repository": "https://hg.mozilla.org/mozilla-central",
+ "head_rev": "abcd",
+ "head_ref": "ef01",
+ "head_tag": "",
+ "message": "",
+ "project": "mozilla-central",
+ "pushlog_id": "143",
+ "pushdate": 1503691511,
+ "owner": "nobody@mozilla.com",
+ "repository_type": "hg",
+ "tasks_for": "hg-push",
+ "level": "3",
+ }
+
+ @patch("gecko_taskgraph.decision.get_hg_revision_branch")
+ @patch("gecko_taskgraph.decision._determine_more_accurate_base_rev")
+ def test_simple_options(
+ self, mock_determine_more_accurate_base_rev, mock_get_hg_revision_branch
+ ):
+ mock_get_hg_revision_branch.return_value = "default"
+ mock_determine_more_accurate_base_rev.return_value = "baserev"
+ with MockedOpen({self.ttc_file: None}):
+ params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options)
+ self.assertEqual(params["pushlog_id"], "143")
+ self.assertEqual(params["build_date"], 1503691511)
+ self.assertEqual(params["hg_branch"], "default")
+ self.assertEqual(params["moz_build_date"], "20170825200511")
+ self.assertEqual(params["try_mode"], None)
+ self.assertEqual(params["try_options"], None)
+ self.assertEqual(params["try_task_config"], {})
+
+ @patch("gecko_taskgraph.decision.get_hg_revision_branch")
+ @patch("gecko_taskgraph.decision._determine_more_accurate_base_rev")
+ def test_no_email_owner(
+ self, mock_determine_more_accurate_base_rev, mock_get_hg_revision_branch
+ ):
+ mock_get_hg_revision_branch.return_value = "default"
+ mock_determine_more_accurate_base_rev.return_value = "baserev"
+ self.options["owner"] = "ffxbld"
+ with MockedOpen({self.ttc_file: None}):
+ params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options)
+ self.assertEqual(params["owner"], "ffxbld@noreply.mozilla.org")
+
+ @patch("gecko_taskgraph.decision.get_hg_revision_branch")
+ @patch("gecko_taskgraph.decision.get_hg_commit_message")
+ @patch("gecko_taskgraph.decision._determine_more_accurate_base_rev")
+ def test_try_options(
+ self,
+ mock_determine_more_accurate_base_rev,
+ mock_get_hg_commit_message,
+ mock_get_hg_revision_branch,
+ ):
+ mock_get_hg_commit_message.return_value = "try: -b do -t all --artifact"
+ mock_get_hg_revision_branch.return_value = "default"
+ mock_determine_more_accurate_base_rev.return_value = "baserev"
+ self.options["project"] = "try"
+ with MockedOpen({self.ttc_file: None}):
+ params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options)
+ self.assertEqual(params["try_mode"], "try_option_syntax")
+ self.assertEqual(params["try_options"]["build_types"], "do")
+ self.assertEqual(params["try_options"]["unittests"], "all")
+ self.assertEqual(
+ params["try_task_config"],
+ {
+ "gecko-profile": False,
+ "use-artifact-builds": True,
+ "env": {},
+ },
+ )
+
+ @patch("gecko_taskgraph.decision.get_hg_revision_branch")
+ @patch("gecko_taskgraph.decision.get_hg_commit_message")
+ @patch("gecko_taskgraph.decision._determine_more_accurate_base_rev")
+ def test_try_task_config(
+ self,
+ mock_get_hg_commit_message,
+ mock_get_hg_revision_branch,
+ mock_determine_more_accurate_base_rev,
+ ):
+ mock_get_hg_commit_message.return_value = "Fuzzy query=foo"
+ mock_get_hg_revision_branch.return_value = "default"
+ mock_determine_more_accurate_base_rev.return_value = "baserev"
+ ttc = {"tasks": ["a", "b"]}
+ self.options["project"] = "try"
+ with MockedOpen({self.ttc_file: json.dumps(ttc)}):
+ params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options)
+ self.assertEqual(params["try_mode"], "try_task_config")
+ self.assertEqual(params["try_options"], None)
+ self.assertEqual(params["try_task_config"], ttc)
+
+ def test_try_syntax_from_message_empty(self):
+ self.assertEqual(decision.try_syntax_from_message(""), "")
+
+ def test_try_syntax_from_message_no_try_syntax(self):
+ self.assertEqual(decision.try_syntax_from_message("abc | def"), "")
+
+ def test_try_syntax_from_message_initial_try_syntax(self):
+ self.assertEqual(
+ decision.try_syntax_from_message("try: -f -o -o"), "try: -f -o -o"
+ )
+
+ def test_try_syntax_from_message_initial_try_syntax_multiline(self):
+ self.assertEqual(
+ decision.try_syntax_from_message("try: -f -o -o\nabc\ndef"), "try: -f -o -o"
+ )
+
+ def test_try_syntax_from_message_embedded_try_syntax_multiline(self):
+ self.assertEqual(
+ decision.try_syntax_from_message("some stuff\ntry: -f -o -o\nabc\ndef"),
+ "try: -f -o -o",
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/taskcluster/gecko_taskgraph/test/test_files_changed.py b/taskcluster/gecko_taskgraph/test/test_files_changed.py
new file mode 100644
index 0000000000..5b9a016649
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/test_files_changed.py
@@ -0,0 +1,90 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import json
+import os
+import unittest
+
+from mozunit import main
+
+from gecko_taskgraph import files_changed
+from gecko_taskgraph.util import hg
+
+PARAMS = {
+ "head_repository": "https://hg.mozilla.org/mozilla-central",
+ "head_rev": "a14f88a9af7a",
+}
+
+FILES_CHANGED = [
+ "devtools/client/debugger/index.html",
+ "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js",
+ "devtools/client/inspector/test/head.js",
+ "devtools/client/themes/rules.css",
+ "devtools/client/webconsole/test/browser_webconsole_output_06.js",
+ "devtools/server/actors/highlighters/eye-dropper.js",
+ "devtools/server/actors/object.js",
+ "docshell/base/nsDocShell.cpp",
+ "dom/tests/mochitest/general/test_contentViewer_overrideDPPX.html",
+ "taskcluster/scripts/builder/build-l10n.sh",
+]
+
+
+class FakeResponse:
+ def json(self):
+ with open(
+ os.path.join(os.path.dirname(__file__), "automationrelevance.json")
+ ) as f:
+ return json.load(f)
+
+
+class TestGetChangedFiles(unittest.TestCase):
+ def setUp(self):
+ files_changed.get_changed_files.clear()
+ self.old_get = hg.requests.get
+
+ def fake_get(url, **kwargs):
+ return FakeResponse()
+
+ hg.requests.get = fake_get
+
+ def tearDown(self):
+ hg.requests.get = self.old_get
+ files_changed.get_changed_files.clear()
+
+ def test_get_changed_files(self):
+ """Get_changed_files correctly gets the list of changed files in a push.
+ This tests against the production hg.mozilla.org so that it will detect
+ any changes in the format of the returned data."""
+ self.assertEqual(
+ sorted(
+ files_changed.get_changed_files(
+ PARAMS["head_repository"], PARAMS["head_rev"]
+ )
+ ),
+ FILES_CHANGED,
+ )
+
+
+class TestCheck(unittest.TestCase):
+ def setUp(self):
+ files_changed.get_changed_files[
+ PARAMS["head_repository"], PARAMS["head_rev"]
+ ] = FILES_CHANGED
+
+ def tearDown(self):
+ files_changed.get_changed_files.clear()
+
+ def test_check_no_params(self):
+ self.assertTrue(files_changed.check({}, ["ignored"]))
+
+ def test_check_no_match(self):
+ self.assertFalse(files_changed.check(PARAMS, ["nosuch/**"]))
+
+ def test_check_match(self):
+ self.assertTrue(files_changed.check(PARAMS, ["devtools/**"]))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/taskcluster/gecko_taskgraph/test/test_main.py b/taskcluster/gecko_taskgraph/test/test_main.py
new file mode 100644
index 0000000000..bb1aa1caeb
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/test_main.py
@@ -0,0 +1,67 @@
+# Any copyright is dedicated to the public domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+import pytest
+from mozunit import main as mozunit_main
+
+import gecko_taskgraph
+from gecko_taskgraph.main import main as taskgraph_main
+
+
+@pytest.fixture
+def run_main(maketgg, monkeypatch):
+ def inner(args, **kwargs):
+ kwargs.setdefault("target_tasks", ["_fake-t-0", "_fake-t-1"])
+ tgg = maketgg(**kwargs)
+
+ def fake_get_taskgraph_generator(*args):
+ return tgg
+
+ monkeypatch.setattr(
+ gecko_taskgraph.main,
+ "get_taskgraph_generator",
+ fake_get_taskgraph_generator,
+ )
+ taskgraph_main(args)
+ return tgg
+
+ return inner
+
+
+@pytest.mark.parametrize(
+ "attr,expected",
+ (
+ ("tasks", ["_fake-t-0", "_fake-t-1", "_fake-t-2"]),
+ ("full", ["_fake-t-0", "_fake-t-1", "_fake-t-2"]),
+ ("target", ["_fake-t-0", "_fake-t-1"]),
+ ("target-graph", ["_fake-t-0", "_fake-t-1"]),
+ ("optimized", ["_fake-t-0", "_fake-t-1"]),
+ ("morphed", ["_fake-t-0", "_fake-t-1"]),
+ ),
+)
+def test_show_taskgraph(run_main, capsys, attr, expected):
+ run_main([attr])
+ out, err = capsys.readouterr()
+ assert out.strip() == "\n".join(expected)
+ assert "Dumping result" in err
+
+
+def test_tasks_regex(run_main, capsys):
+ run_main(["full", "--tasks=_.*-t-1"])
+ out, _ = capsys.readouterr()
+ assert out.strip() == "_fake-t-1"
+
+
+def test_output_file(run_main, tmpdir):
+ output_file = tmpdir.join("out.txt")
+ assert not output_file.check()
+
+ run_main(["full", f"--output-file={output_file.strpath}"])
+ assert output_file.check()
+ assert output_file.read_text("utf-8").strip() == "\n".join(
+ ["_fake-t-0", "_fake-t-1", "_fake-t-2"]
+ )
+
+
+if __name__ == "__main__":
+ mozunit_main()
diff --git a/taskcluster/gecko_taskgraph/test/test_morph.py b/taskcluster/gecko_taskgraph/test/test_morph.py
new file mode 100644
index 0000000000..c29fb58207
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/test_morph.py
@@ -0,0 +1,108 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import pytest
+from mozunit import main
+from taskgraph.graph import Graph
+from taskgraph.parameters import Parameters
+from taskgraph.task import Task
+from taskgraph.taskgraph import TaskGraph
+
+from gecko_taskgraph import morph
+
+
+@pytest.fixture
+def make_taskgraph():
+ def inner(tasks):
+ label_to_taskid = {k: k + "-tid" for k in tasks}
+ for label, task_id in label_to_taskid.items():
+ tasks[label].task_id = task_id
+ graph = Graph(nodes=set(tasks), edges=set())
+ taskgraph = TaskGraph(tasks, graph)
+ return taskgraph, label_to_taskid
+
+ return inner
+
+
+def test_make_index_tasks(make_taskgraph, graph_config):
+ task_def = {
+ "routes": [
+ "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.es-MX",
+ "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.fy-NL",
+ "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.sk",
+ "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.sl",
+ "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.uk",
+ "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.zh-CN",
+ "index.gecko.v2.mozilla-central.pushdate."
+ "2017.04.04.20170404100210.firefox-l10n.linux64-opt.es-MX",
+ "index.gecko.v2.mozilla-central.pushdate."
+ "2017.04.04.20170404100210.firefox-l10n.linux64-opt.fy-NL",
+ "index.gecko.v2.mozilla-central.pushdate."
+ "2017.04.04.20170404100210.firefox-l10n.linux64-opt.sk",
+ "index.gecko.v2.mozilla-central.pushdate."
+ "2017.04.04.20170404100210.firefox-l10n.linux64-opt.sl",
+ "index.gecko.v2.mozilla-central.pushdate."
+ "2017.04.04.20170404100210.firefox-l10n.linux64-opt.uk",
+ "index.gecko.v2.mozilla-central.pushdate."
+ "2017.04.04.20170404100210.firefox-l10n.linux64-opt.zh-CN",
+ "index.gecko.v2.mozilla-central.revision."
+ "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.es-MX",
+ "index.gecko.v2.mozilla-central.revision."
+ "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.fy-NL",
+ "index.gecko.v2.mozilla-central.revision."
+ "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.sk",
+ "index.gecko.v2.mozilla-central.revision."
+ "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.sl",
+ "index.gecko.v2.mozilla-central.revision."
+ "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.uk",
+ "index.gecko.v2.mozilla-central.revision."
+ "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.zh-CN",
+ ],
+ "deadline": "soon",
+ "metadata": {
+ "description": "desc",
+ "owner": "owner@foo.com",
+ "source": "https://source",
+ },
+ "extra": {
+ "index": {"rank": 1540722354},
+ },
+ }
+ task = Task(kind="test", label="a", attributes={}, task=task_def)
+ docker_task = Task(
+ kind="docker-image", label="docker-image-index-task", attributes={}, task={}
+ )
+ taskgraph, label_to_taskid = make_taskgraph(
+ {
+ task.label: task,
+ docker_task.label: docker_task,
+ }
+ )
+
+ index_paths = [
+ r.split(".", 1)[1] for r in task_def["routes"] if r.startswith("index.")
+ ]
+ index_task = morph.make_index_task(
+ task,
+ taskgraph,
+ label_to_taskid,
+ Parameters(strict=False),
+ graph_config,
+ index_paths=index_paths,
+ index_rank=1540722354,
+ purpose="index-task",
+ dependencies={},
+ )
+
+ assert index_task.task["payload"]["command"][0] == "insert-indexes.js"
+ assert index_task.task["payload"]["env"]["TARGET_TASKID"] == "a-tid"
+ assert index_task.task["payload"]["env"]["INDEX_RANK"] == 1540722354
+
+ # check the scope summary
+ assert index_task.task["scopes"] == ["index:insert-task:gecko.v2.mozilla-central.*"]
+
+
+if __name__ == "__main__":
+ main()
diff --git a/taskcluster/gecko_taskgraph/test/test_optimize_strategies.py b/taskcluster/gecko_taskgraph/test/test_optimize_strategies.py
new file mode 100644
index 0000000000..0f37332300
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/test_optimize_strategies.py
@@ -0,0 +1,551 @@
+# Any copyright is dedicated to the public domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+
+import time
+from datetime import datetime
+from time import mktime
+
+import pytest
+from mozunit import main
+from taskgraph.optimize.base import registry
+from taskgraph.task import Task
+
+from gecko_taskgraph.optimize import project
+from gecko_taskgraph.optimize.backstop import SkipUnlessBackstop, SkipUnlessPushInterval
+from gecko_taskgraph.optimize.bugbug import (
+ FALLBACK,
+ BugBugPushSchedules,
+ DisperseGroups,
+ SkipUnlessDebug,
+)
+from gecko_taskgraph.optimize.strategies import IndexSearch, SkipUnlessSchedules
+from gecko_taskgraph.util.backstop import BACKSTOP_PUSH_INTERVAL
+from gecko_taskgraph.util.bugbug import (
+ BUGBUG_BASE_URL,
+ BugbugTimeoutException,
+ push_schedules,
+)
+
+
+@pytest.fixture(autouse=True)
+def clear_push_schedules_memoize():
+ push_schedules.clear()
+
+
+@pytest.fixture
+def params():
+ return {
+ "branch": "autoland",
+ "head_repository": "https://hg.mozilla.org/integration/autoland",
+ "head_rev": "abcdef",
+ "project": "autoland",
+ "pushlog_id": 1,
+ "pushdate": mktime(datetime.now().timetuple()),
+ }
+
+
+def generate_tasks(*tasks):
+ for i, task in enumerate(tasks):
+ task.setdefault("label", f"task-{i}-label")
+ task.setdefault("kind", "test")
+ task.setdefault("task", {})
+ task.setdefault("attributes", {})
+ task["attributes"].setdefault("e10s", True)
+
+ for attr in (
+ "optimization",
+ "dependencies",
+ "soft_dependencies",
+ ):
+ task.setdefault(attr, None)
+
+ task["task"].setdefault("label", task["label"])
+ yield Task.from_json(task)
+
+
+# task sets
+
+default_tasks = list(
+ generate_tasks(
+ {"attributes": {"test_manifests": ["foo/test.ini", "bar/test.ini"]}},
+ {"attributes": {"test_manifests": ["bar/test.ini"], "build_type": "debug"}},
+ {"attributes": {"build_type": "debug"}},
+ {"attributes": {"test_manifests": [], "build_type": "opt"}},
+ {"attributes": {"build_type": "opt"}},
+ )
+)
+
+
+disperse_tasks = list(
+ generate_tasks(
+ {
+ "attributes": {
+ "test_manifests": ["foo/test.ini", "bar/test.ini"],
+ "test_platform": "linux/opt",
+ }
+ },
+ {
+ "attributes": {
+ "test_manifests": ["bar/test.ini"],
+ "test_platform": "linux/opt",
+ }
+ },
+ {
+ "attributes": {
+ "test_manifests": ["bar/test.ini"],
+ "test_platform": "windows/debug",
+ }
+ },
+ {
+ "attributes": {
+ "test_manifests": ["bar/test.ini"],
+ "test_platform": "linux/opt",
+ "unittest_variant": "no-fission",
+ }
+ },
+ {
+ "attributes": {
+ "e10s": False,
+ "test_manifests": ["bar/test.ini"],
+ "test_platform": "linux/opt",
+ }
+ },
+ )
+)
+
+
+def idfn(param):
+ if isinstance(param, tuple):
+ try:
+ return param[0].__name__
+ except AttributeError:
+ return None
+ return None
+
+
+@pytest.mark.parametrize(
+ "opt,tasks,arg,expected",
+ [
+ # debug
+ pytest.param(
+ SkipUnlessDebug(),
+ default_tasks,
+ None,
+ ["task-0-label", "task-1-label", "task-2-label"],
+ ),
+ # disperse with no supplied importance
+ pytest.param(
+ DisperseGroups(),
+ disperse_tasks,
+ None,
+ [t.label for t in disperse_tasks],
+ ),
+ # disperse with low importance
+ pytest.param(
+ DisperseGroups(),
+ disperse_tasks,
+ {"bar/test.ini": "low"},
+ ["task-0-label", "task-2-label"],
+ ),
+ # disperse with medium importance
+ pytest.param(
+ DisperseGroups(),
+ disperse_tasks,
+ {"bar/test.ini": "medium"},
+ ["task-0-label", "task-1-label", "task-2-label"],
+ ),
+ # disperse with high importance
+ pytest.param(
+ DisperseGroups(),
+ disperse_tasks,
+ {"bar/test.ini": "high"},
+ ["task-0-label", "task-1-label", "task-2-label", "task-3-label"],
+ ),
+ ],
+ ids=idfn,
+)
+def test_optimization_strategy_remove(params, opt, tasks, arg, expected):
+ labels = [t.label for t in tasks if not opt.should_remove_task(t, params, arg)]
+ assert sorted(labels) == sorted(expected)
+
+
+@pytest.mark.parametrize(
+ "state,expires,expected",
+ (
+ ("completed", "2021-06-06T14:53:16.937Z", False),
+ ("completed", "2021-06-08T14:53:16.937Z", "abc"),
+ ("exception", "2021-06-08T14:53:16.937Z", False),
+ ("failed", "2021-06-08T14:53:16.937Z", False),
+ ),
+)
+def test_index_search(responses, params, state, expires, expected):
+ taskid = "abc"
+ index_path = "foo.bar.latest"
+ responses.add(
+ responses.GET,
+ f"https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/{index_path}",
+ json={"taskId": taskid},
+ status=200,
+ )
+
+ responses.add(
+ responses.GET,
+ f"https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/{taskid}/status",
+ json={
+ "status": {
+ "state": state,
+ "expires": expires,
+ }
+ },
+ status=200,
+ )
+
+ opt = IndexSearch()
+ deadline = "2021-06-07T19:03:20.482Z"
+ assert opt.should_replace_task({}, params, deadline, (index_path,)) == expected
+
+
+@pytest.mark.parametrize(
+ "args,data,expected",
+ [
+ # empty
+ pytest.param(
+ (0.1,),
+ {},
+ [],
+ ),
+ # only tasks without test manifests selected
+ pytest.param(
+ (0.1,),
+ {"tasks": {"task-1-label": 0.9, "task-2-label": 0.1, "task-3-label": 0.5}},
+ ["task-2-label"],
+ ),
+ # tasks which are unknown to bugbug are selected
+ pytest.param(
+ (0.1,),
+ {
+ "tasks": {"task-1-label": 0.9, "task-3-label": 0.5},
+ "known_tasks": ["task-1-label", "task-3-label", "task-4-label"],
+ },
+ ["task-2-label"],
+ ),
+ # tasks containing groups selected
+ pytest.param(
+ (0.1,),
+ {"groups": {"foo/test.ini": 0.4}},
+ ["task-0-label"],
+ ),
+ # tasks matching "tasks" or "groups" selected
+ pytest.param(
+ (0.1,),
+ {
+ "tasks": {"task-2-label": 0.2},
+ "groups": {"foo/test.ini": 0.25, "bar/test.ini": 0.75},
+ },
+ ["task-0-label", "task-1-label", "task-2-label"],
+ ),
+ # tasks matching "tasks" or "groups" selected, when they exceed the confidence threshold
+ pytest.param(
+ (0.5,),
+ {
+ "tasks": {"task-2-label": 0.2, "task-4-label": 0.5},
+ "groups": {"foo/test.ini": 0.65, "bar/test.ini": 0.25},
+ },
+ ["task-0-label", "task-4-label"],
+ ),
+ # tasks matching "reduced_tasks" are selected, when they exceed the confidence threshold
+ pytest.param(
+ (0.7, True, True),
+ {
+ "tasks": {"task-2-label": 0.7, "task-4-label": 0.7},
+ "reduced_tasks": {"task-4-label": 0.7},
+ "groups": {"foo/test.ini": 0.75, "bar/test.ini": 0.25},
+ },
+ ["task-4-label"],
+ ),
+ # tasks matching "groups" selected, only on specific platforms.
+ pytest.param(
+ (0.1, False, False, None, 1, True),
+ {
+ "tasks": {"task-2-label": 0.2},
+ "groups": {"foo/test.ini": 0.25, "bar/test.ini": 0.75},
+ "config_groups": {
+ "foo/test.ini": ["task-1-label", "task-0-label"],
+ "bar/test.ini": ["task-0-label"],
+ },
+ },
+ ["task-0-label", "task-2-label"],
+ ),
+ pytest.param(
+ (0.1, False, False, None, 1, True),
+ {
+ "tasks": {"task-2-label": 0.2},
+ "groups": {"foo/test.ini": 0.25, "bar/test.ini": 0.75},
+ "config_groups": {
+ "foo/test.ini": ["task-1-label", "task-0-label"],
+ "bar/test.ini": ["task-1-label"],
+ },
+ },
+ ["task-0-label", "task-1-label", "task-2-label"],
+ ),
+ pytest.param(
+ (0.1, False, False, None, 1, True),
+ {
+ "tasks": {"task-2-label": 0.2},
+ "groups": {"foo/test.ini": 0.25, "bar/test.ini": 0.75},
+ "config_groups": {
+ "foo/test.ini": ["task-1-label"],
+ "bar/test.ini": ["task-0-label"],
+ },
+ },
+ ["task-0-label", "task-2-label"],
+ ),
+ pytest.param(
+ (0.1, False, False, None, 1, True),
+ {
+ "tasks": {"task-2-label": 0.2},
+ "groups": {"foo/test.ini": 0.25, "bar/test.ini": 0.75},
+ "config_groups": {
+ "foo/test.ini": ["task-1-label"],
+ "bar/test.ini": ["task-3-label"],
+ },
+ },
+ ["task-2-label"],
+ ),
+ ],
+ ids=idfn,
+)
+def test_bugbug_push_schedules(responses, params, args, data, expected):
+ query = "/push/{branch}/{head_rev}/schedules".format(**params)
+ url = BUGBUG_BASE_URL + query
+
+ responses.add(
+ responses.GET,
+ url,
+ json=data,
+ status=200,
+ )
+
+ opt = BugBugPushSchedules(*args)
+ labels = [
+ t.label for t in default_tasks if not opt.should_remove_task(t, params, {})
+ ]
+ assert sorted(labels) == sorted(expected)
+
+
+def test_bugbug_multiple_pushes(responses, params):
+ pushes = {str(pid): {"changesets": [f"c{pid}"]} for pid in range(8, 10)}
+
+ responses.add(
+ responses.GET,
+ "https://hg.mozilla.org/integration/autoland/json-pushes/?version=2&startID=8&endID=9",
+ json={"pushes": pushes},
+ status=200,
+ )
+
+ responses.add(
+ responses.GET,
+ BUGBUG_BASE_URL + "/push/{}/c9/schedules".format(params["branch"]),
+ json={
+ "tasks": {"task-2-label": 0.2, "task-4-label": 0.5},
+ "groups": {"foo/test.ini": 0.2, "bar/test.ini": 0.25},
+ "config_groups": {"foo/test.ini": ["linux-*"], "bar/test.ini": ["task-*"]},
+ "known_tasks": ["task-4-label"],
+ },
+ status=200,
+ )
+
+ # Tasks with a lower confidence don't override task with a higher one.
+ # Tasks with a higher confidence override tasks with a lower one.
+ # Known tasks are merged.
+ responses.add(
+ responses.GET,
+ BUGBUG_BASE_URL + "/push/{branch}/{head_rev}/schedules".format(**params),
+ json={
+ "tasks": {"task-2-label": 0.2, "task-4-label": 0.2},
+ "groups": {"foo/test.ini": 0.65, "bar/test.ini": 0.25},
+ "config_groups": {
+ "foo/test.ini": ["task-*"],
+ "bar/test.ini": ["windows-*"],
+ },
+ "known_tasks": ["task-1-label", "task-3-label"],
+ },
+ status=200,
+ )
+
+ params["pushlog_id"] = 10
+
+ opt = BugBugPushSchedules(0.3, False, False, False, 2)
+ labels = [
+ t.label for t in default_tasks if not opt.should_remove_task(t, params, {})
+ ]
+ assert sorted(labels) == sorted(["task-0-label", "task-2-label", "task-4-label"])
+
+ opt = BugBugPushSchedules(0.3, False, False, False, 2, True)
+ labels = [
+ t.label for t in default_tasks if not opt.should_remove_task(t, params, {})
+ ]
+ assert sorted(labels) == sorted(["task-0-label", "task-2-label", "task-4-label"])
+
+ opt = BugBugPushSchedules(0.2, False, False, False, 2, True)
+ labels = [
+ t.label for t in default_tasks if not opt.should_remove_task(t, params, {})
+ ]
+ assert sorted(labels) == sorted(
+ ["task-0-label", "task-1-label", "task-2-label", "task-4-label"]
+ )
+
+
+def test_bugbug_timeout(monkeypatch, responses, params):
+ query = "/push/{branch}/{head_rev}/schedules".format(**params)
+ url = BUGBUG_BASE_URL + query
+ responses.add(
+ responses.GET,
+ url,
+ json={"ready": False},
+ status=202,
+ )
+
+ # Make sure the test runs fast.
+ monkeypatch.setattr(time, "sleep", lambda i: None)
+
+ opt = BugBugPushSchedules(0.5)
+ with pytest.raises(BugbugTimeoutException):
+ opt.should_remove_task(default_tasks[0], params, None)
+
+
+def test_bugbug_fallback(monkeypatch, responses, params):
+ query = "/push/{branch}/{head_rev}/schedules".format(**params)
+ url = BUGBUG_BASE_URL + query
+ responses.add(
+ responses.GET,
+ url,
+ json={"ready": False},
+ status=202,
+ )
+
+ opt = BugBugPushSchedules(0.5, fallback=FALLBACK)
+
+ # Make sure the test runs fast.
+ monkeypatch.setattr(time, "sleep", lambda i: None)
+
+ def fake_should_remove_task(task, params, _):
+ return task.label == default_tasks[0].label
+
+ monkeypatch.setattr(
+ registry[FALLBACK], "should_remove_task", fake_should_remove_task
+ )
+
+ assert opt.should_remove_task(default_tasks[0], params, None)
+
+ # Make sure we don't hit bugbug more than once.
+ responses.reset()
+
+ assert not opt.should_remove_task(default_tasks[1], params, None)
+
+
+def test_backstop(params):
+ all_labels = {t.label for t in default_tasks}
+ opt = SkipUnlessBackstop()
+
+ params["backstop"] = False
+ scheduled = {
+ t.label for t in default_tasks if not opt.should_remove_task(t, params, None)
+ }
+ assert scheduled == set()
+
+ params["backstop"] = True
+ scheduled = {
+ t.label for t in default_tasks if not opt.should_remove_task(t, params, None)
+ }
+ assert scheduled == all_labels
+
+
+def test_push_interval(params):
+ all_labels = {t.label for t in default_tasks}
+ opt = SkipUnlessPushInterval(10) # every 10th push
+
+ # Only multiples of 10 schedule tasks.
+ params["pushlog_id"] = 9
+ scheduled = {
+ t.label for t in default_tasks if not opt.should_remove_task(t, params, None)
+ }
+ assert scheduled == set()
+
+ params["pushlog_id"] = 10
+ scheduled = {
+ t.label for t in default_tasks if not opt.should_remove_task(t, params, None)
+ }
+ assert scheduled == all_labels
+
+
+def test_expanded(params):
+ all_labels = {t.label for t in default_tasks}
+ opt = registry["skip-unless-expanded"]
+
+ params["backstop"] = False
+ params["pushlog_id"] = BACKSTOP_PUSH_INTERVAL / 2
+ scheduled = {
+ t.label for t in default_tasks if not opt.should_remove_task(t, params, None)
+ }
+ assert scheduled == all_labels
+
+ params["pushlog_id"] += 1
+ scheduled = {
+ t.label for t in default_tasks if not opt.should_remove_task(t, params, None)
+ }
+ assert scheduled == set()
+
+ params["backstop"] = True
+ scheduled = {
+ t.label for t in default_tasks if not opt.should_remove_task(t, params, None)
+ }
+ assert scheduled == all_labels
+
+
+def test_project_autoland_test(monkeypatch, responses, params):
+ """Tests the behaviour of the `project.autoland["test"]` strategy on
+ various types of pushes.
+ """
+ # This is meant to test the composition of substrategies, and not the
+ # actual optimization implementations. So mock them out for simplicity.
+ monkeypatch.setattr(SkipUnlessSchedules, "should_remove_task", lambda *args: False)
+ monkeypatch.setattr(DisperseGroups, "should_remove_task", lambda *args: False)
+
+ def fake_bugbug_should_remove_task(self, task, params, importance):
+ if self.num_pushes > 1:
+ return task.label == "task-4-label"
+ return task.label in ("task-2-label", "task-3-label", "task-4-label")
+
+ monkeypatch.setattr(
+ BugBugPushSchedules, "should_remove_task", fake_bugbug_should_remove_task
+ )
+
+ opt = project.autoland["test"]
+
+ # On backstop pushes, nothing gets optimized.
+ params["backstop"] = True
+ scheduled = {
+ t.label for t in default_tasks if not opt.should_remove_task(t, params, {})
+ }
+ assert scheduled == {t.label for t in default_tasks}
+
+ # On expanded pushes, some things are optimized.
+ params["backstop"] = False
+ params["pushlog_id"] = 10
+ scheduled = {
+ t.label for t in default_tasks if not opt.should_remove_task(t, params, {})
+ }
+ assert scheduled == {"task-0-label", "task-1-label", "task-2-label", "task-3-label"}
+
+ # On regular pushes, more things are optimized.
+ params["pushlog_id"] = 11
+ scheduled = {
+ t.label for t in default_tasks if not opt.should_remove_task(t, params, {})
+ }
+ assert scheduled == {"task-0-label", "task-1-label"}
+
+
+if __name__ == "__main__":
+ main()
diff --git a/taskcluster/gecko_taskgraph/test/test_target_tasks.py b/taskcluster/gecko_taskgraph/test/test_target_tasks.py
new file mode 100644
index 0000000000..154c0f910a
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/test_target_tasks.py
@@ -0,0 +1,371 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import contextlib
+import re
+import unittest
+
+import pytest
+from mozunit import main
+from taskgraph.graph import Graph
+from taskgraph.task import Task
+from taskgraph.taskgraph import TaskGraph
+
+from gecko_taskgraph import target_tasks, try_option_syntax
+
+
+class FakeTryOptionSyntax:
+ def __init__(self, message, task_graph, graph_config):
+ self.trigger_tests = 0
+ self.talos_trigger_tests = 0
+ self.raptor_trigger_tests = 0
+ self.notifications = None
+ self.env = []
+ self.profile = False
+ self.tag = None
+ self.no_retry = False
+
+ def task_matches(self, task):
+ return "at-at" in task.attributes
+
+
+class TestTargetTasks(unittest.TestCase):
+ def default_matches_project(self, run_on_projects, project):
+ return self.default_matches(
+ attributes={
+ "run_on_projects": run_on_projects,
+ },
+ parameters={
+ "project": project,
+ "hg_branch": "default",
+ },
+ )
+
+ def default_matches_hg_branch(self, run_on_hg_branches, hg_branch):
+ attributes = {"run_on_projects": ["all"]}
+ if run_on_hg_branches is not None:
+ attributes["run_on_hg_branches"] = run_on_hg_branches
+
+ return self.default_matches(
+ attributes=attributes,
+ parameters={
+ "project": "mozilla-central",
+ "hg_branch": hg_branch,
+ },
+ )
+
+ def default_matches(self, attributes, parameters):
+ method = target_tasks.get_method("default")
+ graph = TaskGraph(
+ tasks={
+ "a": Task(kind="build", label="a", attributes=attributes, task={}),
+ },
+ graph=Graph(nodes={"a"}, edges=set()),
+ )
+ return "a" in method(graph, parameters, {})
+
+ def test_default_all(self):
+ """run_on_projects=[all] includes release, integration, and other projects"""
+ self.assertTrue(self.default_matches_project(["all"], "mozilla-central"))
+ self.assertTrue(self.default_matches_project(["all"], "baobab"))
+
+ def test_default_integration(self):
+ """run_on_projects=[integration] includes integration projects"""
+ self.assertFalse(
+ self.default_matches_project(["integration"], "mozilla-central")
+ )
+ self.assertFalse(self.default_matches_project(["integration"], "baobab"))
+
+ def test_default_release(self):
+ """run_on_projects=[release] includes release projects"""
+ self.assertTrue(self.default_matches_project(["release"], "mozilla-central"))
+ self.assertFalse(self.default_matches_project(["release"], "baobab"))
+
+ def test_default_nothing(self):
+ """run_on_projects=[] includes nothing"""
+ self.assertFalse(self.default_matches_project([], "mozilla-central"))
+ self.assertFalse(self.default_matches_project([], "baobab"))
+
+ def test_default_hg_branch(self):
+ self.assertTrue(self.default_matches_hg_branch(None, "default"))
+ self.assertTrue(self.default_matches_hg_branch(None, "GECKOVIEW_62_RELBRANCH"))
+
+ self.assertFalse(self.default_matches_hg_branch([], "default"))
+ self.assertFalse(self.default_matches_hg_branch([], "GECKOVIEW_62_RELBRANCH"))
+
+ self.assertTrue(self.default_matches_hg_branch(["all"], "default"))
+ self.assertTrue(
+ self.default_matches_hg_branch(["all"], "GECKOVIEW_62_RELBRANCH")
+ )
+
+ self.assertTrue(self.default_matches_hg_branch(["default"], "default"))
+ self.assertTrue(self.default_matches_hg_branch([r"default"], "default"))
+ self.assertFalse(
+ self.default_matches_hg_branch([r"default"], "GECKOVIEW_62_RELBRANCH")
+ )
+
+ self.assertTrue(
+ self.default_matches_hg_branch(
+ ["GECKOVIEW_62_RELBRANCH"], "GECKOVIEW_62_RELBRANCH"
+ )
+ )
+ self.assertTrue(
+ self.default_matches_hg_branch(
+ [r"GECKOVIEW_\d+_RELBRANCH"], "GECKOVIEW_62_RELBRANCH"
+ )
+ )
+ self.assertTrue(
+ self.default_matches_hg_branch(
+ [r"GECKOVIEW_\d+_RELBRANCH"], "GECKOVIEW_62_RELBRANCH"
+ )
+ )
+ self.assertFalse(
+ self.default_matches_hg_branch([r"GECKOVIEW_\d+_RELBRANCH"], "default")
+ )
+
+ def make_task_graph(self):
+ tasks = {
+ "a": Task(kind=None, label="a", attributes={}, task={}),
+ "b": Task(kind=None, label="b", attributes={"at-at": "yep"}, task={}),
+ "c": Task(
+ kind=None, label="c", attributes={"run_on_projects": ["try"]}, task={}
+ ),
+ }
+ graph = Graph(nodes=set("abc"), edges=set())
+ return TaskGraph(tasks, graph)
+
+ @contextlib.contextmanager
+ def fake_TryOptionSyntax(self):
+ orig_TryOptionSyntax = try_option_syntax.TryOptionSyntax
+ try:
+ try_option_syntax.TryOptionSyntax = FakeTryOptionSyntax
+ yield
+ finally:
+ try_option_syntax.TryOptionSyntax = orig_TryOptionSyntax
+
+ def test_empty_try(self):
+ "try_mode = None runs nothing"
+ tg = self.make_task_graph()
+ method = target_tasks.get_method("try_tasks")
+ params = {
+ "try_mode": None,
+ "project": "try",
+ "message": "",
+ }
+ # only runs the task with run_on_projects: try
+ self.assertEqual(method(tg, params, {}), [])
+
+ def test_try_option_syntax(self):
+ "try_mode = try_option_syntax uses TryOptionSyntax"
+ tg = self.make_task_graph()
+ method = target_tasks.get_method("try_tasks")
+ with self.fake_TryOptionSyntax():
+ params = {
+ "try_mode": "try_option_syntax",
+ "message": "try: -p all",
+ }
+ self.assertEqual(method(tg, params, {}), ["b"])
+
+ def test_try_task_config(self):
+ "try_mode = try_task_config uses the try config"
+ tg = self.make_task_graph()
+ method = target_tasks.get_method("try_tasks")
+ params = {
+ "try_mode": "try_task_config",
+ "try_task_config": {"tasks": ["a"]},
+ }
+ self.assertEqual(method(tg, params, {}), ["a"])
+
+
+# tests for specific filters
+
+
+@pytest.mark.parametrize(
+ "name,params,expected",
+ (
+ pytest.param(
+ "filter_tests_without_manifests",
+ {
+ "task": Task(kind="test", label="a", attributes={}, task={}),
+ "parameters": None,
+ },
+ True,
+ id="filter_tests_without_manifests_not_in_attributes",
+ ),
+ pytest.param(
+ "filter_tests_without_manifests",
+ {
+ "task": Task(
+ kind="test",
+ label="a",
+ attributes={"test_manifests": ["foo"]},
+ task={},
+ ),
+ "parameters": None,
+ },
+ True,
+ id="filter_tests_without_manifests_has_test_manifests",
+ ),
+ pytest.param(
+ "filter_tests_without_manifests",
+ {
+ "task": Task(
+ kind="build",
+ label="a",
+ attributes={"test_manifests": None},
+ task={},
+ ),
+ "parameters": None,
+ },
+ True,
+ id="filter_tests_without_manifests_not_a_test",
+ ),
+ pytest.param(
+ "filter_tests_without_manifests",
+ {
+ "task": Task(
+ kind="test", label="a", attributes={"test_manifests": None}, task={}
+ ),
+ "parameters": None,
+ },
+ False,
+ id="filter_tests_without_manifests_has_no_test_manifests",
+ ),
+ pytest.param(
+ "filter_by_regex",
+ {
+ "task_label": "build-linux64-debug",
+ "regexes": [re.compile("build")],
+ "mode": "include",
+ },
+ True,
+ id="filter_regex_simple_include",
+ ),
+ pytest.param(
+ "filter_by_regex",
+ {
+ "task_label": "build-linux64-debug",
+ "regexes": [re.compile("linux(.+)debug")],
+ "mode": "include",
+ },
+ True,
+ id="filter_regex_re_include",
+ ),
+ pytest.param(
+ "filter_by_regex",
+ {
+ "task_label": "build-linux64-debug",
+ "regexes": [re.compile("nothing"), re.compile("linux(.+)debug")],
+ "mode": "include",
+ },
+ True,
+ id="filter_regex_re_include_multiple",
+ ),
+ pytest.param(
+ "filter_by_regex",
+ {
+ "task_label": "build-linux64-debug",
+ "regexes": [re.compile("build")],
+ "mode": "exclude",
+ },
+ False,
+ id="filter_regex_simple_exclude",
+ ),
+ pytest.param(
+ "filter_by_regex",
+ {
+ "task_label": "build-linux64-debug",
+ "regexes": [re.compile("linux(.+)debug")],
+ "mode": "exclude",
+ },
+ False,
+ id="filter_regex_re_exclude",
+ ),
+ pytest.param(
+ "filter_by_regex",
+ {
+ "task_label": "build-linux64-debug",
+ "regexes": [re.compile("linux(.+)debug"), re.compile("nothing")],
+ "mode": "exclude",
+ },
+ False,
+ id="filter_regex_re_exclude_multiple",
+ ),
+ pytest.param(
+ "filter_unsupported_artifact_builds",
+ {
+ "task": Task(
+ kind="test",
+ label="a",
+ attributes={"supports-artifact-builds": False},
+ task={},
+ ),
+ "parameters": {
+ "try_task_config": {
+ "use-artifact-builds": False,
+ },
+ },
+ },
+ True,
+ id="filter_unsupported_artifact_builds_no_artifact_builds",
+ ),
+ pytest.param(
+ "filter_unsupported_artifact_builds",
+ {
+ "task": Task(
+ kind="test",
+ label="a",
+ attributes={"supports-artifact-builds": False},
+ task={},
+ ),
+ "parameters": {
+ "try_task_config": {
+ "use-artifact-builds": True,
+ },
+ },
+ },
+ False,
+ id="filter_unsupported_artifact_builds_removed",
+ ),
+ pytest.param(
+ "filter_unsupported_artifact_builds",
+ {
+ "task": Task(
+ kind="test",
+ label="a",
+ attributes={"supports-artifact-builds": True},
+ task={},
+ ),
+ "parameters": {
+ "try_task_config": {
+ "use-artifact-builds": True,
+ },
+ },
+ },
+ True,
+ id="filter_unsupported_artifact_builds_not_removed",
+ ),
+ pytest.param(
+ "filter_unsupported_artifact_builds",
+ {
+ "task": Task(kind="test", label="a", attributes={}, task={}),
+ "parameters": {
+ "try_task_config": {
+ "use-artifact-builds": True,
+ },
+ },
+ },
+ True,
+ id="filter_unsupported_artifact_builds_not_removed",
+ ),
+ ),
+)
+def test_filters(name, params, expected):
+ func = getattr(target_tasks, name)
+ assert func(**params) is expected
+
+
+if __name__ == "__main__":
+ main()
diff --git a/taskcluster/gecko_taskgraph/test/test_taskcluster_yml.py b/taskcluster/gecko_taskgraph/test/test_taskcluster_yml.py
new file mode 100644
index 0000000000..cdf94ec3e1
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/test_taskcluster_yml.py
@@ -0,0 +1,145 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import pprint
+import unittest
+
+import jsone
+import slugid
+from mozunit import main
+from taskgraph.util.time import current_json_time
+from taskgraph.util.yaml import load_yaml
+
+from gecko_taskgraph import GECKO
+
+
+class TestTaskclusterYml(unittest.TestCase):
+ @property
+ def taskcluster_yml(self):
+ return load_yaml(GECKO, ".taskcluster.yml")
+
+ def test_push(self):
+ context = {
+ "tasks_for": "hg-push",
+ "push": {
+ "revision": "e8d2d9aff5026ef1f1777b781b47fdcbdb9d8f20",
+ "base_revision": "e8aebe488b2f2e567940577de25013d00e818f7c",
+ "owner": "dustin@mozilla.com",
+ "pushlog_id": 1556565286,
+ "pushdate": 112957,
+ },
+ "repository": {
+ "url": "https://hg.mozilla.org/mozilla-central",
+ "project": "mozilla-central",
+ "level": "3",
+ },
+ "ownTaskId": slugid.nice(),
+ }
+ rendered = jsone.render(self.taskcluster_yml, context)
+ pprint.pprint(rendered)
+ self.assertEqual(
+ rendered["tasks"][0]["metadata"]["name"], "Gecko Decision Task"
+ )
+ self.assertIn("matrixBody", rendered["tasks"][0]["extra"]["notify"])
+
+ def test_push_non_mc(self):
+ context = {
+ "tasks_for": "hg-push",
+ "push": {
+ "revision": "e8d2d9aff5026ef1f1777b781b47fdcbdb9d8f20",
+ "base_revision": "e8aebe488b2f2e567940577de25013d00e818f7c",
+ "owner": "dustin@mozilla.com",
+ "pushlog_id": 1556565286,
+ "pushdate": 112957,
+ },
+ "repository": {
+ "url": "https://hg.mozilla.org/releases/mozilla-beta",
+ "project": "mozilla-beta",
+ "level": "3",
+ },
+ "ownTaskId": slugid.nice(),
+ }
+ rendered = jsone.render(self.taskcluster_yml, context)
+ pprint.pprint(rendered)
+ self.assertEqual(
+ rendered["tasks"][0]["metadata"]["name"], "Gecko Decision Task"
+ )
+ self.assertNotIn("matrixBody", rendered["tasks"][0]["extra"]["notify"])
+
+ def test_cron(self):
+ context = {
+ "tasks_for": "cron",
+ "repository": {
+ "url": "https://hg.mozilla.org/mozilla-central",
+ "project": "mozilla-central",
+ "level": 3,
+ },
+ "push": {
+ "revision": "e8aebe488b2f2e567940577de25013d00e818f7c",
+ "base_revision": "54cbb3745cdb9a8aa0a4428d405b3b2e1c7d13c2",
+ "pushlog_id": -1,
+ "pushdate": 0,
+ "owner": "cron",
+ },
+ "cron": {
+ "task_id": "<cron task id>",
+ "job_name": "test",
+ "job_symbol": "T",
+ "quoted_args": "abc def",
+ },
+ "now": current_json_time(),
+ "ownTaskId": slugid.nice(),
+ }
+ rendered = jsone.render(self.taskcluster_yml, context)
+ pprint.pprint(rendered)
+ self.assertEqual(
+ rendered["tasks"][0]["metadata"]["name"], "Decision Task for cron job test"
+ )
+
+ def test_action(self):
+ context = {
+ "tasks_for": "action",
+ "repository": {
+ "url": "https://hg.mozilla.org/mozilla-central",
+ "project": "mozilla-central",
+ "level": 3,
+ },
+ "push": {
+ "revision": "e8d2d9aff5026ef1f1777b781b47fdcbdb9d8f20",
+ "base_revision": "e8aebe488b2f2e567940577de25013d00e818f7c",
+ "owner": "dustin@mozilla.com",
+ "pushlog_id": 1556565286,
+ "pushdate": 112957,
+ },
+ "action": {
+ "name": "test-action",
+ "title": "Test Action",
+ "description": "Just testing",
+ "taskGroupId": slugid.nice(),
+ "symbol": "t",
+ "repo_scope": "assume:repo:hg.mozilla.org/try:action:generic",
+ "cb_name": "test_action",
+ },
+ "input": {},
+ "parameters": {},
+ "now": current_json_time(),
+ "taskId": slugid.nice(),
+ "ownTaskId": slugid.nice(),
+ "clientId": "testing/testing/testing",
+ }
+ rendered = jsone.render(self.taskcluster_yml, context)
+ pprint.pprint(rendered)
+ self.assertEqual(
+ rendered["tasks"][0]["metadata"]["name"], "Action: Test Action"
+ )
+
+ def test_unknown(self):
+ context = {"tasks_for": "bitkeeper-push"}
+ rendered = jsone.render(self.taskcluster_yml, context)
+ pprint.pprint(rendered)
+ self.assertEqual(rendered["tasks"], [])
+
+
+if __name__ == "__main__":
+ main()
diff --git a/taskcluster/gecko_taskgraph/test/test_transforms_job.py b/taskcluster/gecko_taskgraph/test/test_transforms_job.py
new file mode 100644
index 0000000000..8bf85dcf62
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/test_transforms_job.py
@@ -0,0 +1,150 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Tests for the 'job' transform subsystem.
+"""
+
+
+import os
+from copy import deepcopy
+
+import pytest
+from mozunit import main
+from taskgraph.config import load_graph_config
+from taskgraph.transforms.base import TransformConfig
+from taskgraph.util.schema import Schema, validate_schema
+
+from gecko_taskgraph import GECKO
+from gecko_taskgraph.test.conftest import FakeParameters
+from gecko_taskgraph.transforms import job
+from gecko_taskgraph.transforms.job import run_task # noqa: F401
+from gecko_taskgraph.transforms.job.common import add_cache
+from gecko_taskgraph.transforms.task import payload_builders
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+TASK_DEFAULTS = {
+ "description": "fake description",
+ "label": "fake-task-label",
+ "run": {
+ "using": "run-task",
+ },
+}
+
+
+@pytest.fixture(scope="module")
+def config():
+ graph_config = load_graph_config(os.path.join(GECKO, "taskcluster", "ci"))
+ params = FakeParameters(
+ {
+ "base_repository": "http://hg.example.com",
+ "head_repository": "http://hg.example.com",
+ "head_rev": "abcdef",
+ "level": 1,
+ "project": "example",
+ }
+ )
+ return TransformConfig(
+ "job_test", here, {}, params, {}, graph_config, write_artifacts=False
+ )
+
+
+@pytest.fixture()
+def transform(monkeypatch, config):
+ """Run the job transforms on the specified task but return the inputs to
+ `configure_taskdesc_for_run` without executing it.
+
+ This gives test functions an easy way to generate the inputs required for
+ many of the `run_using` subsystems.
+ """
+
+ def inner(task_input):
+ task = deepcopy(TASK_DEFAULTS)
+ task.update(task_input)
+ frozen_args = []
+
+ def _configure_taskdesc_for_run(*args):
+ frozen_args.extend(args)
+
+ monkeypatch.setattr(
+ job, "configure_taskdesc_for_run", _configure_taskdesc_for_run
+ )
+
+ for _ in job.transforms(config, [task]):
+ # This forces the generator to be evaluated
+ pass
+
+ return frozen_args
+
+ return inner
+
+
+@pytest.mark.parametrize(
+ "task",
+ [
+ {"worker-type": "b-linux"},
+ {"worker-type": "t-win10-64-hw"},
+ ],
+ ids=lambda t: t["worker-type"],
+)
+def test_worker_caches(task, transform):
+ config, job, taskdesc, impl = transform(task)
+ add_cache(job, taskdesc, "cache1", "/cache1")
+ add_cache(job, taskdesc, "cache2", "/cache2", skip_untrusted=True)
+
+ if impl not in ("docker-worker", "generic-worker"):
+ pytest.xfail(f"caches not implemented for '{impl}'")
+
+ key = "caches" if impl == "docker-worker" else "mounts"
+ assert key in taskdesc["worker"]
+ assert len(taskdesc["worker"][key]) == 2
+
+ # Create a new schema object with just the part relevant to caches.
+ partial_schema = Schema(payload_builders[impl].schema.schema[key])
+ validate_schema(partial_schema, taskdesc["worker"][key], "validation error")
+
+
+@pytest.mark.parametrize(
+ "workerfn", [fn for fn, *_ in job.registry["run-task"].values()]
+)
+@pytest.mark.parametrize(
+ "task",
+ (
+ {
+ "worker-type": "b-linux",
+ "run": {
+ "checkout": True,
+ "comm-checkout": False,
+ "command": "echo '{output}'",
+ "command-context": {"output": "hello", "extra": None},
+ "run-as-root": False,
+ "sparse-profile": False,
+ "tooltool-downloads": False,
+ },
+ },
+ ),
+)
+def test_run_task_command_context(task, transform, workerfn):
+ config, job_, taskdesc, _ = transform(task)
+ job_ = deepcopy(job_)
+
+ def assert_cmd(expected):
+ cmd = taskdesc["worker"]["command"]
+ while isinstance(cmd, list):
+ cmd = cmd[-1]
+ assert cmd == expected
+
+ workerfn(config, job_, taskdesc)
+ assert_cmd("echo 'hello'")
+
+ job_copy = job_.copy()
+ del job_copy["run"]["command-context"]
+ workerfn(config, job_copy, taskdesc)
+ assert_cmd("echo '{output}'")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/taskcluster/gecko_taskgraph/test/test_transforms_test.py b/taskcluster/gecko_taskgraph/test/test_transforms_test.py
new file mode 100644
index 0000000000..2b90fed3e7
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/test_transforms_test.py
@@ -0,0 +1,328 @@
+# Any copyright is dedicated to the Public Domain.
+# https://creativecommons.org/publicdomain/zero/1.0/
+"""
+Tests for the 'tests.py' transforms
+"""
+
+import hashlib
+import json
+from functools import partial
+from pprint import pprint
+
+import mozunit
+import pytest
+
+from gecko_taskgraph.transforms import test as test_transforms
+
+
+@pytest.fixture
+def make_test_task():
+ """Create a test task definition with required default values."""
+
+ def inner(**extra):
+ task = {
+ "attributes": {},
+ "build-platform": "linux64",
+ "mozharness": {"extra-options": []},
+ "test-platform": "linux64",
+ "treeherder-symbol": "g(t)",
+ "try-name": "task",
+ }
+ task.update(extra)
+ return task
+
+ return inner
+
+
+def test_split_variants(monkeypatch, run_full_config_transform, make_test_task):
+ # mock out variant definitions
+ monkeypatch.setattr(
+ test_transforms.variant,
+ "TEST_VARIANTS",
+ {
+ "foo": {
+ "description": "foo variant",
+ "suffix": "foo",
+ "component": "foo bar",
+ "expiration": "never",
+ "merge": {
+ "mozharness": {
+ "extra-options": [
+ "--setpref=foo=1",
+ ],
+ },
+ },
+ },
+ "bar": {
+ "description": "bar variant",
+ "suffix": "bar",
+ "component": "foo bar",
+ "expiration": "never",
+ "when": {
+ "$eval": "task['test-platform'][:5] == 'linux'",
+ },
+ "merge": {
+ "mozharness": {
+ "extra-options": [
+ "--setpref=bar=1",
+ ],
+ },
+ },
+ "replace": {"tier": 2},
+ },
+ },
+ )
+
+ def make_expected(variant):
+ """Helper to generate expected tasks."""
+ return make_test_task(
+ **{
+ "attributes": {"unittest_variant": variant},
+ "description": f"{variant} variant",
+ "mozharness": {
+ "extra-options": [f"--setpref={variant}=1"],
+ },
+ "treeherder-symbol": f"g-{variant}(t)",
+ "variant-suffix": f"-{variant}",
+ }
+ )
+
+ run_split_variants = partial(
+ run_full_config_transform, test_transforms.variant.split_variants
+ )
+
+ # test no variants
+ input_task = make_test_task(
+ **{
+ "run-without-variant": True,
+ }
+ )
+ tasks = list(run_split_variants(input_task))
+ assert len(tasks) == 1
+ assert tasks[0] == input_task
+
+ # test variants are split into expected tasks
+ input_task = make_test_task(
+ **{
+ "run-without-variant": True,
+ "variants": ["foo", "bar"],
+ }
+ )
+ tasks = list(run_split_variants(input_task))
+ assert len(tasks) == 3
+ assert tasks[0] == make_test_task()
+ assert tasks[1] == make_expected("foo")
+
+ expected = make_expected("bar")
+ expected["tier"] = 2
+ assert tasks[2] == expected
+
+ # test composite variants
+ input_task = make_test_task(
+ **{
+ "run-without-variant": True,
+ "variants": ["foo+bar"],
+ }
+ )
+ tasks = list(run_split_variants(input_task))
+ assert len(tasks) == 2
+ assert tasks[1]["attributes"]["unittest_variant"] == "foo+bar"
+ assert tasks[1]["mozharness"]["extra-options"] == [
+ "--setpref=foo=1",
+ "--setpref=bar=1",
+ ]
+ assert tasks[1]["treeherder-symbol"] == "g-foo-bar(t)"
+
+ # test 'when' filter
+ input_task = make_test_task(
+ **{
+ "run-without-variant": True,
+ # this should cause task to be filtered out of 'bar' and 'foo+bar' variants
+ "test-platform": "windows",
+ "variants": ["foo", "bar", "foo+bar"],
+ }
+ )
+ tasks = list(run_split_variants(input_task))
+ assert len(tasks) == 2
+ assert "unittest_variant" not in tasks[0]["attributes"]
+ assert tasks[1]["attributes"]["unittest_variant"] == "foo"
+
+ # test 'run-without-variants=False'
+ input_task = make_test_task(
+ **{
+ "run-without-variant": False,
+ "variants": ["foo"],
+ }
+ )
+ tasks = list(run_split_variants(input_task))
+ assert len(tasks) == 1
+ assert tasks[0]["attributes"]["unittest_variant"] == "foo"
+
+
+@pytest.mark.parametrize(
+ "task,expected",
+ (
+ pytest.param(
+ {
+ "attributes": {"unittest_variant": "webrender-sw+1proc"},
+ "test-platform": "linux1804-64-clang-trunk-qr/opt",
+ },
+ {
+ "platform": {
+ "arch": "64",
+ "os": {
+ "name": "linux",
+ "version": "1804",
+ },
+ },
+ "build": {
+ "type": "opt",
+ "clang-trunk": True,
+ },
+ "runtime": {
+ "1proc": True,
+ "webrender-sw": True,
+ },
+ },
+ id="linux",
+ ),
+ pytest.param(
+ {
+ "attributes": {},
+ "test-platform": "linux2204-64-wayland-shippable/opt",
+ },
+ {
+ "platform": {
+ "arch": "64",
+ "display": "wayland",
+ "os": {
+ "name": "linux",
+ "version": "2204",
+ },
+ },
+ "build": {
+ "type": "opt",
+ "shippable": True,
+ },
+ "runtime": {},
+ },
+ id="linux wayland shippable",
+ ),
+ pytest.param(
+ {
+ "attributes": {},
+ "test-platform": "android-hw-a51-11-0-arm7-shippable-qr/opt",
+ },
+ {
+ "platform": {
+ "arch": "arm7",
+ "device": "a51",
+ "os": {
+ "name": "android",
+ "version": "11.0",
+ },
+ },
+ "build": {
+ "type": "opt",
+ "shippable": True,
+ },
+ "runtime": {},
+ },
+ id="android",
+ ),
+ pytest.param(
+ {
+ "attributes": {},
+ "test-platform": "windows10-64-2004-ref-hw-2017-ccov/debug",
+ },
+ {
+ "platform": {
+ "arch": "64",
+ "machine": "ref-hw-2017",
+ "os": {
+ "build": "2004",
+ "name": "windows",
+ "version": "10",
+ },
+ },
+ "build": {
+ "type": "debug",
+ "ccov": True,
+ },
+ "runtime": {},
+ },
+ id="windows",
+ ),
+ ),
+)
+def test_set_test_setting(run_transform, task, expected):
+ # add hash to 'expected'
+ expected["_hash"] = hashlib.sha256(
+ json.dumps(expected, sort_keys=True).encode("utf-8")
+ ).hexdigest()[:12]
+
+ task = list(run_transform(test_transforms.other.set_test_setting, task))[0]
+ assert "test-setting" in task
+ assert task["test-setting"] == expected
+
+
+def assert_spi_not_disabled(task):
+ extra_options = task["mozharness"]["extra-options"]
+ # The pref to enable this gets set outside of this transform, so only
+ # bother asserting that the pref to disable does not exist.
+ assert (
+ "--setpref=media.peerconnection.mtransport_process=false" not in extra_options
+ )
+ assert "--setpref=network.process.enabled=false" not in extra_options
+
+
+def assert_spi_disabled(task):
+ extra_options = task["mozharness"]["extra-options"]
+ assert "--setpref=media.peerconnection.mtransport_process=false" in extra_options
+ assert "--setpref=media.peerconnection.mtransport_process=true" not in extra_options
+ assert "--setpref=network.process.enabled=false" in extra_options
+ assert "--setpref=network.process.enabled=true" not in extra_options
+
+
+@pytest.mark.parametrize(
+ "task,callback",
+ (
+ pytest.param(
+ {"attributes": {"unittest_variant": "socketprocess"}},
+ assert_spi_not_disabled,
+ id="socketprocess",
+ ),
+ pytest.param(
+ {
+ "attributes": {"unittest_variant": "socketprocess_networking"},
+ },
+ assert_spi_not_disabled,
+ id="socketprocess_networking",
+ ),
+ pytest.param({}, assert_spi_disabled, id="no variant"),
+ pytest.param(
+ {"suite": "cppunit", "attributes": {"unittest_variant": "socketprocess"}},
+ assert_spi_not_disabled,
+ id="excluded suite",
+ ),
+ pytest.param(
+ {"attributes": {"unittest_variant": "no-fission+socketprocess"}},
+ assert_spi_not_disabled,
+ id="composite variant",
+ ),
+ ),
+)
+def test_ensure_spi_disabled_on_all_but_spi(
+ make_test_task, run_transform, task, callback
+):
+ task.setdefault("suite", "mochitest-plain")
+ task = make_test_task(**task)
+ task = list(
+ run_transform(test_transforms.other.ensure_spi_disabled_on_all_but_spi, task)
+ )[0]
+ pprint(task)
+ callback(task)
+
+
+if __name__ == "__main__":
+ mozunit.main()
diff --git a/taskcluster/gecko_taskgraph/test/test_try_option_syntax.py b/taskcluster/gecko_taskgraph/test/test_try_option_syntax.py
new file mode 100644
index 0000000000..a37de53378
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/test_try_option_syntax.py
@@ -0,0 +1,430 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import unittest
+
+from mozunit import main
+from taskgraph.graph import Graph
+from taskgraph.task import Task
+from taskgraph.taskgraph import TaskGraph
+
+from gecko_taskgraph.try_option_syntax import TryOptionSyntax, parse_message
+
+
+def unittest_task(n, tp, bt="opt"):
+ return (
+ n,
+ Task(
+ "test",
+ n,
+ {
+ "unittest_try_name": n,
+ "test_platform": tp.split("/")[0],
+ "build_type": bt,
+ },
+ {},
+ ),
+ )
+
+
+def talos_task(n, tp, bt="opt"):
+ return (
+ n,
+ Task(
+ "test",
+ n,
+ {
+ "talos_try_name": n,
+ "test_platform": tp.split("/")[0],
+ "build_type": bt,
+ },
+ {},
+ ),
+ )
+
+
+tasks = {
+ k: v
+ for k, v in [
+ unittest_task("mochitest-browser-chrome", "linux/opt"),
+ unittest_task("mochitest-browser-chrome-e10s", "linux64/opt"),
+ unittest_task("mochitest-chrome", "linux/debug", "debug"),
+ unittest_task("mochitest-webgl1-core", "linux/debug", "debug"),
+ unittest_task("mochitest-webgl1-ext", "linux/debug", "debug"),
+ unittest_task("mochitest-webgl2-core", "linux/debug", "debug"),
+ unittest_task("mochitest-webgl2-ext", "linux/debug", "debug"),
+ unittest_task("mochitest-webgl2-deqp", "linux/debug", "debug"),
+ unittest_task("extra1", "linux", "debug/opt"),
+ unittest_task("extra2", "win32/opt"),
+ unittest_task("crashtest-e10s", "linux/other"),
+ unittest_task("gtest", "linux64/asan"),
+ talos_task("dromaeojs", "linux64/psan"),
+ unittest_task("extra3", "linux/opt"),
+ unittest_task("extra4", "linux64/debug", "debug"),
+ unittest_task("extra5", "linux/this"),
+ unittest_task("extra6", "linux/that"),
+ unittest_task("extra7", "linux/other"),
+ unittest_task("extra8", "linux64/asan"),
+ talos_task("extra9", "linux64/psan"),
+ ]
+}
+
+RIDEALONG_BUILDS = {
+ "linux": ["linux-ridealong"],
+ "linux64": ["linux64-ridealong"],
+}
+
+GRAPH_CONFIG = {
+ "try": {"ridealong-builds": RIDEALONG_BUILDS},
+}
+
+for r in RIDEALONG_BUILDS.values():
+ tasks.update({k: v for k, v in [unittest_task(n + "-test", n) for n in r]})
+
+unittest_tasks = {k: v for k, v in tasks.items() if "unittest_try_name" in v.attributes}
+talos_tasks = {k: v for k, v in tasks.items() if "talos_try_name" in v.attributes}
+graph_with_jobs = TaskGraph(tasks, Graph(set(tasks), set()))
+
+
+class TestTryOptionSyntax(unittest.TestCase):
+ def test_unknown_args(self):
+ "unknown arguments are ignored"
+ parameters = parse_message("try: --doubledash -z extra")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ # equilvant to "try:"..
+ self.assertEqual(tos.build_types, [])
+ self.assertEqual(tos.jobs, [])
+
+ def test_apostrophe_in_message(self):
+ "apostrophe does not break parsing"
+ parameters = parse_message("Increase spammy log's log level. try: -b do")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(sorted(tos.build_types), ["debug", "opt"])
+
+ def test_b_do(self):
+ "-b do should produce both build_types"
+ parameters = parse_message("try: -b do")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(sorted(tos.build_types), ["debug", "opt"])
+
+ def test_b_d(self):
+ "-b d should produce build_types=['debug']"
+ parameters = parse_message("try: -b d")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(sorted(tos.build_types), ["debug"])
+
+ def test_b_o(self):
+ "-b o should produce build_types=['opt']"
+ parameters = parse_message("try: -b o")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(sorted(tos.build_types), ["opt"])
+
+ def test_build_o(self):
+ "--build o should produce build_types=['opt']"
+ parameters = parse_message("try: --build o")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(sorted(tos.build_types), ["opt"])
+
+ def test_b_dx(self):
+ "-b dx should produce build_types=['debug'], silently ignoring the x"
+ parameters = parse_message("try: -b dx")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(sorted(tos.build_types), ["debug"])
+
+ def test_j_job(self):
+ "-j somejob sets jobs=['somejob']"
+ parameters = parse_message("try: -j somejob")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(sorted(tos.jobs), ["somejob"])
+
+ def test_j_jobs(self):
+ "-j job1,job2 sets jobs=['job1', 'job2']"
+ parameters = parse_message("try: -j job1,job2")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(sorted(tos.jobs), ["job1", "job2"])
+
+ def test_j_all(self):
+ "-j all sets jobs=None"
+ parameters = parse_message("try: -j all")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.jobs, None)
+
+ def test_j_twice(self):
+ "-j job1 -j job2 sets jobs=job1, job2"
+ parameters = parse_message("try: -j job1 -j job2")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(sorted(tos.jobs), sorted(["job1", "job2"]))
+
+ def test_p_all(self):
+ "-p all sets platforms=None"
+ parameters = parse_message("try: -p all")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.platforms, None)
+
+ def test_p_linux(self):
+ "-p linux sets platforms=['linux', 'linux-ridealong']"
+ parameters = parse_message("try: -p linux")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.platforms, ["linux", "linux-ridealong"])
+
+ def test_p_linux_win32(self):
+ "-p linux,win32 sets platforms=['linux', 'linux-ridealong', 'win32']"
+ parameters = parse_message("try: -p linux,win32")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(sorted(tos.platforms), ["linux", "linux-ridealong", "win32"])
+
+ def test_p_expands_ridealongs(self):
+ "-p linux,linux64 includes the RIDEALONG_BUILDS"
+ parameters = parse_message("try: -p linux,linux64")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ platforms = set(["linux"] + RIDEALONG_BUILDS["linux"])
+ platforms |= set(["linux64"] + RIDEALONG_BUILDS["linux64"])
+ self.assertEqual(sorted(tos.platforms), sorted(platforms))
+
+ def test_u_none(self):
+ "-u none sets unittests=[]"
+ parameters = parse_message("try: -u none")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.unittests, [])
+
+ def test_u_all(self):
+ "-u all sets unittests=[..whole list..]"
+ parameters = parse_message("try: -u all")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.unittests, [{"test": t} for t in sorted(unittest_tasks)])
+
+ def test_u_single(self):
+ "-u mochitest-webgl1-core sets unittests=[mochitest-webgl1-core]"
+ parameters = parse_message("try: -u mochitest-webgl1-core")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.unittests, [{"test": "mochitest-webgl1-core"}])
+
+ def test_u_alias(self):
+ "-u mochitest-gl sets unittests=[mochitest-webgl*]"
+ parameters = parse_message("try: -u mochitest-gl")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(
+ tos.unittests,
+ [
+ {"test": t}
+ for t in [
+ "mochitest-webgl1-core",
+ "mochitest-webgl1-ext",
+ "mochitest-webgl2-core",
+ "mochitest-webgl2-deqp",
+ "mochitest-webgl2-ext",
+ ]
+ ],
+ )
+
+ def test_u_multi_alias(self):
+ "-u e10s sets unittests=[all e10s unittests]"
+ parameters = parse_message("try: -u e10s")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(
+ tos.unittests, [{"test": t} for t in sorted(unittest_tasks) if "e10s" in t]
+ )
+
+ def test_u_commas(self):
+ "-u mochitest-webgl1-core,gtest sets unittests=both"
+ parameters = parse_message("try: -u mochitest-webgl1-core,gtest")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(
+ tos.unittests,
+ [
+ {"test": "gtest"},
+ {"test": "mochitest-webgl1-core"},
+ ],
+ )
+
+ def test_u_chunks(self):
+ "-u gtest-3,gtest-4 selects the third and fourth chunk of gtest"
+ parameters = parse_message("try: -u gtest-3,gtest-4")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(
+ sorted(tos.unittests),
+ sorted(
+ [
+ {"test": "gtest", "only_chunks": set("34")},
+ ]
+ ),
+ )
+
+ def test_u_platform(self):
+ "-u gtest[linux] selects the linux platform for gtest"
+ parameters = parse_message("try: -u gtest[linux]")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(
+ sorted(tos.unittests),
+ sorted(
+ [
+ {"test": "gtest", "platforms": ["linux"]},
+ ]
+ ),
+ )
+
+ def test_u_platforms(self):
+ "-u gtest[linux,win32] selects the linux and win32 platforms for gtest"
+ parameters = parse_message("try: -u gtest[linux,win32]")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(
+ sorted(tos.unittests),
+ sorted(
+ [
+ {"test": "gtest", "platforms": ["linux", "win32"]},
+ ]
+ ),
+ )
+
+ def test_u_platforms_pretty(self):
+ """-u gtest[Ubuntu] selects the linux, linux64 and linux64-asan
+ platforms for gtest"""
+ parameters = parse_message("try: -u gtest[Ubuntu]")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(
+ sorted(tos.unittests),
+ sorted(
+ [
+ {
+ "test": "gtest",
+ "platforms": [
+ "linux32",
+ "linux64",
+ "linux64-asan",
+ "linux1804-64",
+ "linux1804-64-asan",
+ ],
+ },
+ ]
+ ),
+ )
+
+ def test_u_platforms_negated(self):
+ "-u gtest[-linux] selects all platforms but linux for gtest"
+ parameters = parse_message("try: -u gtest[-linux]")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ all_platforms = {x.attributes["test_platform"] for x in unittest_tasks.values()}
+ self.assertEqual(
+ sorted(tos.unittests[0]["platforms"]),
+ sorted(x for x in all_platforms if x != "linux"),
+ )
+
+ def test_u_platforms_negated_pretty(self):
+ "-u gtest[Ubuntu,-x64] selects just linux for gtest"
+ parameters = parse_message("try: -u gtest[Ubuntu,-x64]")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(
+ sorted(tos.unittests),
+ sorted(
+ [
+ {"test": "gtest", "platforms": ["linux32"]},
+ ]
+ ),
+ )
+
+ def test_u_chunks_platforms(self):
+ "-u gtest-1[linux,win32] selects the linux and win32 platforms for chunk 1 of gtest"
+ parameters = parse_message("try: -u gtest-1[linux,win32]")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(
+ tos.unittests,
+ [
+ {
+ "test": "gtest",
+ "platforms": ["linux", "win32"],
+ "only_chunks": set("1"),
+ },
+ ],
+ )
+
+ def test_t_none(self):
+ "-t none sets talos=[]"
+ parameters = parse_message("try: -t none")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.talos, [])
+
+ def test_t_all(self):
+ "-t all sets talos=[..whole list..]"
+ parameters = parse_message("try: -t all")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.talos, [{"test": t} for t in sorted(talos_tasks)])
+
+ def test_t_single(self):
+ "-t mochitest-webgl sets talos=[mochitest-webgl]"
+ parameters = parse_message("try: -t mochitest-webgl")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.talos, [{"test": "mochitest-webgl"}])
+
+ # -t shares an implementation with -u, so it's not tested heavily
+
+ def test_trigger_tests(self):
+ "--rebuild 10 sets trigger_tests"
+ parameters = parse_message("try: --rebuild 10")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.trigger_tests, 10)
+
+ def test_talos_trigger_tests(self):
+ "--rebuild-talos 10 sets talos_trigger_tests"
+ parameters = parse_message("try: --rebuild-talos 10")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.talos_trigger_tests, 10)
+
+ def test_interactive(self):
+ "--interactive sets interactive"
+ parameters = parse_message("try: --interactive")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.interactive, True)
+
+ def test_all_email(self):
+ "--all-emails sets notifications"
+ parameters = parse_message("try: --all-emails")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.notifications, "all")
+
+ def test_fail_email(self):
+ "--failure-emails sets notifications"
+ parameters = parse_message("try: --failure-emails")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.notifications, "failure")
+
+ def test_no_email(self):
+ "no email settings don't set notifications"
+ parameters = parse_message("try:")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.notifications, None)
+
+ def test_setenv(self):
+ "--setenv VAR=value adds a environment variables setting to env"
+ parameters = parse_message("try: --setenv VAR1=value1 --setenv VAR2=value2")
+ assert parameters["try_task_config"]["env"] == {
+ "VAR1": "value1",
+ "VAR2": "value2",
+ }
+
+ def test_profile(self):
+ "--gecko-profile sets profile to true"
+ parameters = parse_message("try: --gecko-profile")
+ assert parameters["try_task_config"]["gecko-profile"] is True
+
+ def test_tag(self):
+ "--tag TAG sets tag to TAG value"
+ parameters = parse_message("try: --tag tagName")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.tag, "tagName")
+
+ def test_no_retry(self):
+ "--no-retry sets no_retry to true"
+ parameters = parse_message("try: --no-retry")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertTrue(tos.no_retry)
+
+ def test_artifact(self):
+ "--artifact sets artifact to true"
+ parameters = parse_message("try: --artifact")
+ assert parameters["try_task_config"]["use-artifact-builds"] is True
+
+
+if __name__ == "__main__":
+ main()
diff --git a/taskcluster/gecko_taskgraph/test/test_util_attributes.py b/taskcluster/gecko_taskgraph/test/test_util_attributes.py
new file mode 100644
index 0000000000..26181cb8bf
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/test_util_attributes.py
@@ -0,0 +1,99 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import unittest
+
+from mozunit import main
+from taskgraph.util.attributes import attrmatch
+
+from gecko_taskgraph.util.attributes import match_run_on_projects
+
+
+class Attrmatch(unittest.TestCase):
+ def test_trivial_match(self):
+ """Given no conditions, anything matches"""
+ self.assertTrue(attrmatch({}))
+
+ def test_missing_attribute(self):
+ """If a filtering attribute is not present, no match"""
+ self.assertFalse(attrmatch({}, someattr=10))
+
+ def test_literal_attribute(self):
+ """Literal attributes must match exactly"""
+ self.assertTrue(attrmatch({"att": 10}, att=10))
+ self.assertFalse(attrmatch({"att": 10}, att=20))
+
+ def test_set_attribute(self):
+ """Set attributes require set membership"""
+ self.assertTrue(attrmatch({"att": 10}, att={9, 10}))
+ self.assertFalse(attrmatch({"att": 10}, att={19, 20}))
+
+ def test_callable_attribute(self):
+ """Callable attributes are called and any False causes the match to fail"""
+ self.assertTrue(attrmatch({"att": 10}, att=lambda val: True))
+ self.assertFalse(attrmatch({"att": 10}, att=lambda val: False))
+
+ def even(val):
+ return val % 2 == 0
+
+ self.assertTrue(attrmatch({"att": 10}, att=even))
+ self.assertFalse(attrmatch({"att": 11}, att=even))
+
+ def test_all_matches_required(self):
+ """If only one attribute does not match, the result is False"""
+ self.assertFalse(attrmatch({"a": 1}, a=1, b=2, c=3))
+ self.assertFalse(attrmatch({"a": 1, "b": 2}, a=1, b=2, c=3))
+ self.assertTrue(attrmatch({"a": 1, "b": 2, "c": 3}, a=1, b=2, c=3))
+
+
+class MatchRunOnProjects(unittest.TestCase):
+ def test_empty(self):
+ self.assertFalse(match_run_on_projects("birch", []))
+
+ def test_all(self):
+ self.assertTrue(match_run_on_projects("birch", ["all"]))
+ self.assertTrue(match_run_on_projects("larch", ["all"]))
+ self.assertTrue(match_run_on_projects("autoland", ["all"]))
+ self.assertTrue(match_run_on_projects("mozilla-central", ["all"]))
+ self.assertTrue(match_run_on_projects("mozilla-beta", ["all"]))
+ self.assertTrue(match_run_on_projects("mozilla-release", ["all"]))
+
+ def test_release(self):
+ self.assertFalse(match_run_on_projects("birch", ["release"]))
+ self.assertFalse(match_run_on_projects("larch", ["release"]))
+ self.assertFalse(match_run_on_projects("autoland", ["release"]))
+ self.assertTrue(match_run_on_projects("mozilla-central", ["release"]))
+ self.assertTrue(match_run_on_projects("mozilla-beta", ["release"]))
+ self.assertTrue(match_run_on_projects("mozilla-release", ["release"]))
+
+ def test_integration(self):
+ self.assertFalse(match_run_on_projects("birch", ["integration"]))
+ self.assertFalse(match_run_on_projects("larch", ["integration"]))
+ self.assertTrue(match_run_on_projects("autoland", ["integration"]))
+ self.assertFalse(match_run_on_projects("mozilla-central", ["integration"]))
+ self.assertFalse(match_run_on_projects("mozilla-beta", ["integration"]))
+ self.assertFalse(match_run_on_projects("mozilla-integration", ["integration"]))
+
+ def test_combo(self):
+ self.assertTrue(match_run_on_projects("birch", ["release", "birch", "maple"]))
+ self.assertFalse(match_run_on_projects("larch", ["release", "birch", "maple"]))
+ self.assertTrue(match_run_on_projects("maple", ["release", "birch", "maple"]))
+ self.assertFalse(
+ match_run_on_projects("autoland", ["release", "birch", "maple"])
+ )
+ self.assertTrue(
+ match_run_on_projects("mozilla-central", ["release", "birch", "maple"])
+ )
+ self.assertTrue(
+ match_run_on_projects("mozilla-beta", ["release", "birch", "maple"])
+ )
+ self.assertTrue(
+ match_run_on_projects("mozilla-release", ["release", "birch", "maple"])
+ )
+ self.assertTrue(match_run_on_projects("birch", ["birch", "trunk"]))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/taskcluster/gecko_taskgraph/test/test_util_backstop.py b/taskcluster/gecko_taskgraph/test/test_util_backstop.py
new file mode 100644
index 0000000000..af9aabd5af
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/test_util_backstop.py
@@ -0,0 +1,155 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+from datetime import datetime
+from textwrap import dedent
+from time import mktime
+
+import pytest
+from mozunit import main
+from taskgraph.util.taskcluster import get_artifact_url, get_index_url, get_task_url
+
+from gecko_taskgraph.util.backstop import (
+ BACKSTOP_INDEX,
+ BACKSTOP_PUSH_INTERVAL,
+ BACKSTOP_TIME_INTERVAL,
+ is_backstop,
+)
+
+LAST_BACKSTOP_ID = 0
+LAST_BACKSTOP_PUSHDATE = mktime(datetime.now().timetuple())
+DEFAULT_RESPONSES = {
+ "index": {
+ "status": 200,
+ "json": {"taskId": LAST_BACKSTOP_ID},
+ },
+ "artifact": {
+ "status": 200,
+ "body": dedent(
+ """
+ pushdate: {}
+ """.format(
+ LAST_BACKSTOP_PUSHDATE
+ )
+ ),
+ },
+ "status": {
+ "status": 200,
+ "json": {"status": {"state": "complete"}},
+ },
+}
+
+
+@pytest.fixture
+def params():
+ return {
+ "branch": "integration/autoland",
+ "head_repository": "https://hg.mozilla.org/integration/autoland",
+ "head_rev": "abcdef",
+ "project": "autoland",
+ "pushdate": LAST_BACKSTOP_PUSHDATE + 1,
+ "pushlog_id": LAST_BACKSTOP_ID + 1,
+ }
+
+
+@pytest.mark.parametrize(
+ "response_args,extra_params,expected",
+ (
+ pytest.param(
+ {
+ "index": {"status": 404},
+ },
+ {"pushlog_id": 1},
+ True,
+ id="no previous backstop",
+ ),
+ pytest.param(
+ {
+ "index": DEFAULT_RESPONSES["index"],
+ "status": DEFAULT_RESPONSES["status"],
+ "artifact": {"status": 404},
+ },
+ {"pushlog_id": 1},
+ False,
+ id="previous backstop not finished",
+ ),
+ pytest.param(
+ DEFAULT_RESPONSES,
+ {
+ "pushlog_id": LAST_BACKSTOP_ID + 1,
+ "pushdate": LAST_BACKSTOP_PUSHDATE + 1,
+ },
+ False,
+ id="not a backstop",
+ ),
+ pytest.param(
+ {},
+ {
+ "pushlog_id": BACKSTOP_PUSH_INTERVAL,
+ },
+ True,
+ id="backstop interval",
+ ),
+ pytest.param(
+ DEFAULT_RESPONSES,
+ {
+ "pushdate": LAST_BACKSTOP_PUSHDATE + (BACKSTOP_TIME_INTERVAL * 60),
+ },
+ True,
+ id="time elapsed",
+ ),
+ pytest.param(
+ {},
+ {
+ "project": "try",
+ "pushlog_id": BACKSTOP_PUSH_INTERVAL,
+ },
+ False,
+ id="try not a backstop",
+ ),
+ pytest.param(
+ {},
+ {
+ "project": "mozilla-central",
+ },
+ True,
+ id="release branches always a backstop",
+ ),
+ pytest.param(
+ {
+ "index": DEFAULT_RESPONSES["index"],
+ "status": {
+ "status": 200,
+ "json": {"status": {"state": "failed"}},
+ },
+ },
+ {},
+ True,
+ id="last backstop failed",
+ ),
+ ),
+)
+def test_is_backstop(responses, params, response_args, extra_params, expected):
+ urls = {
+ "index": get_index_url(
+ BACKSTOP_INDEX.format(
+ **{"trust-domain": "gecko", "project": params["project"]}
+ )
+ ),
+ "artifact": get_artifact_url(LAST_BACKSTOP_ID, "public/parameters.yml"),
+ "status": get_task_url(LAST_BACKSTOP_ID) + "/status",
+ }
+
+ for key in ("index", "status", "artifact"):
+ if key in response_args:
+ print(urls[key])
+ responses.add(responses.GET, urls[key], **response_args[key])
+
+ params.update(extra_params)
+ assert is_backstop(params) is expected
+
+
+if __name__ == "__main__":
+ main()
diff --git a/taskcluster/gecko_taskgraph/test/test_util_bugbug.py b/taskcluster/gecko_taskgraph/test/test_util_bugbug.py
new file mode 100644
index 0000000000..7e8865ddde
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/test_util_bugbug.py
@@ -0,0 +1,57 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import mozunit
+
+from gecko_taskgraph.util.bugbug import BUGBUG_BASE_URL, push_schedules
+
+
+def test_group_translation(responses):
+ branch = ("integration/autoland",)
+ rev = "abcdef"
+ query = f"/push/{branch}/{rev}/schedules"
+ url = BUGBUG_BASE_URL + query
+
+ responses.add(
+ responses.GET,
+ url,
+ json={
+ "groups": {
+ "dom/indexedDB": 1,
+ "testing/web-platform/tests/IndexedDB": 1,
+ "testing/web-platform/mozilla/tests/IndexedDB": 1,
+ },
+ "config_groups": {
+ "dom/indexedDB": ["label1", "label2"],
+ "testing/web-platform/tests/IndexedDB": ["label3"],
+ "testing/web-platform/mozilla/tests/IndexedDB": ["label4"],
+ },
+ },
+ status=200,
+ )
+
+ assert len(push_schedules) == 0
+ data = push_schedules(branch, rev)
+ print(data)
+ assert sorted(data["groups"]) == [
+ "/IndexedDB",
+ "/_mozilla/IndexedDB",
+ "dom/indexedDB",
+ ]
+ assert data["config_groups"] == {
+ "dom/indexedDB": ["label1", "label2"],
+ "/IndexedDB": ["label3"],
+ "/_mozilla/IndexedDB": ["label4"],
+ }
+ assert len(push_schedules) == 1
+
+ # Value is memoized.
+ responses.reset()
+ push_schedules(branch, rev)
+ assert len(push_schedules) == 1
+
+
+if __name__ == "__main__":
+ mozunit.main()
diff --git a/taskcluster/gecko_taskgraph/test/test_util_chunking.py b/taskcluster/gecko_taskgraph/test/test_util_chunking.py
new file mode 100644
index 0000000000..99434af042
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/test_util_chunking.py
@@ -0,0 +1,403 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import re
+from itertools import combinations
+
+import pytest
+from mozunit import main
+
+from gecko_taskgraph.util import chunking
+
+pytestmark = pytest.mark.slow
+
+
+@pytest.fixture(scope="module")
+def mock_manifest_runtimes():
+ """Deterministically produce a list of simulated manifest runtimes.
+
+ Args:
+ manifests (list): list of manifests against which simulated manifest
+ runtimes would be paired up to.
+
+ Returns:
+ dict of manifest data paired with a float value representing runtime.
+ """
+
+ def inner(manifests):
+ manifests = sorted(manifests)
+ # Generate deterministic runtime data.
+ runtimes = [(i / 10) ** (i / 10) for i in range(len(manifests))]
+ return dict(zip(manifests, runtimes))
+
+ return inner
+
+
+@pytest.fixture(scope="module")
+def unchunked_manifests():
+ """Produce a list of unchunked manifests to be consumed by test method.
+
+ Args:
+ length (int, optional): number of path elements to keep.
+ cutoff (int, optional): number of generated test paths to remove
+ from the test set if user wants to limit the number of paths.
+
+ Returns:
+ list: list of test paths.
+ """
+ data = ["blueberry", "nashi", "peach", "watermelon"]
+
+ def inner(suite, length=2, cutoff=0):
+ if "web-platform" in suite:
+ suffix = ""
+ prefix = "/"
+ elif "reftest" in suite:
+ suffix = ".list"
+ prefix = ""
+ else:
+ suffix = ".ini"
+ prefix = ""
+ return [prefix + "/".join(p) + suffix for p in combinations(data, length)][
+ cutoff:
+ ]
+
+ return inner
+
+
+@pytest.fixture(scope="module")
+def mock_task_definition():
+ """Builds a mock task definition for use in testing.
+
+ Args:
+ os_name (str): represents the os.
+ os_version (str): represents the os version
+ bits (int): software bits.
+ build_type (str): opt or debug.
+ build_attrs (list, optional): specify build attribute(s)
+ variants (list, optional): specify runtime variant(s)
+
+ Returns:
+ dict: mocked task definition.
+ """
+
+ def inner(os_name, os_version, bits, build_type, build_attrs=None, variants=None):
+ setting = {
+ "platform": {
+ "arch": str(bits),
+ "os": {
+ "name": os_name,
+ "version": os_version,
+ },
+ },
+ "build": {
+ "type": build_type,
+ },
+ "runtime": {},
+ }
+
+ # Optionally set build attributes and runtime variants.
+ if build_attrs:
+ if isinstance(build_attrs, str):
+ build_attrs = [build_attrs]
+ for attr in build_attrs:
+ setting["build"][attr] = True
+
+ if variants:
+ if isinstance(variants, str):
+ variants = [variants]
+ for variant in variants:
+ setting["runtime"][variant] = True
+ return {"test-name": "foo", "test-setting": setting}
+
+ return inner
+
+
+@pytest.fixture(scope="module")
+def mock_mozinfo():
+ """Returns a mocked mozinfo object, similar to guess_mozinfo_from_task().
+
+ Args:
+ os (str): typically one of 'win, linux, mac, android'.
+ processor (str): processor architecture.
+ asan (bool, optional): addressanitizer build.
+ bits (int, optional): defaults to 64.
+ ccov (bool, optional): code coverage build.
+ debug (bool, optional): debug type build.
+ fission (bool, optional): process fission.
+ headless (bool, optional): headless browser testing without displays.
+ tsan (bool, optional): threadsanitizer build.
+
+ Returns:
+ dict: Dictionary mimickign the results from guess_mozinfo_from_task.
+ """
+
+ def inner(
+ os,
+ processor,
+ asan=False,
+ bits=64,
+ ccov=False,
+ debug=False,
+ fission=False,
+ headless=False,
+ tsan=False,
+ ):
+ return {
+ "os": os,
+ "processor": processor,
+ "toolkit": "",
+ "asan": asan,
+ "bits": bits,
+ "ccov": ccov,
+ "debug": debug,
+ "e10s": True,
+ "fission": fission,
+ "headless": headless,
+ "tsan": tsan,
+ "appname": "firefox",
+ "condprof": False,
+ }
+
+ return inner
+
+
+@pytest.mark.parametrize(
+ "params,exception",
+ [
+ [("win", "7", 32, "opt"), None],
+ [("win", "10", 64, "opt"), None],
+ [("linux", "1804", 64, "debug"), None],
+ [("macosx", "1015", 64, "debug"), None],
+ [("macosx", "1100", 64, "opt"), None],
+ [("android", "", 64, "debug"), None],
+ [("and", "", 64, "debug"), ValueError],
+ [("", "", 64, "opt"), ValueError],
+ [("linux", "1804", 64, "opt", ["ccov"]), None],
+ [("linux", "1804", 64, "opt", ["asan"]), None],
+ [("win", "10", 64, "opt", ["tsan"]), None],
+ [("mac", "1100", 64, "opt", ["ccov"]), None],
+ [("android", "", 64, "opt", None, ["fission"]), None],
+ [("win", "10", "aarch64", "opt"), None],
+ ],
+)
+def test_guess_mozinfo_from_task(params, exception, mock_task_definition):
+ """Tests the mozinfo guessing process."""
+ # Set up a mocked task object.
+ task = mock_task_definition(*params)
+
+ if exception:
+ with pytest.raises(exception):
+ result = chunking.guess_mozinfo_from_task(task)
+ else:
+ expected_toolkits = {
+ "android": "android",
+ "linux": "gtk",
+ "mac": "cocoa",
+ "win": "windows",
+ }
+ result = chunking.guess_mozinfo_from_task(task)
+ setting = task["test-setting"]
+
+ assert str(result["bits"]) in setting["platform"]["arch"]
+ assert result["os"] in ("android", "linux", "mac", "win")
+ assert result["os"] in setting["platform"]["os"]["name"]
+ assert result["toolkit"] == expected_toolkits[result["os"]]
+
+ # Ensure the outcome of special build variants being present match what
+ # guess_mozinfo_from_task method returns for these attributes.
+ assert ("asan" in setting["build"]) == result["asan"]
+ assert ("tsan" in setting["build"]) == result["tsan"]
+ assert ("ccov" in setting["build"]) == result["ccov"]
+
+ # Ensure runtime variants match
+ assert ("fission" in setting["runtime"]) == result["fission"]
+ assert ("1proc" in setting["runtime"]) != result["e10s"]
+
+
+@pytest.mark.parametrize("platform", ["unix", "windows", "android"])
+@pytest.mark.parametrize(
+ "suite", ["crashtest", "reftest", "web-platform-tests", "xpcshell"]
+)
+def test_get_runtimes(platform, suite):
+ """Tests that runtime information is returned for known good configurations."""
+ assert chunking.get_runtimes(platform, suite)
+
+
+@pytest.mark.parametrize(
+ "platform,suite,exception",
+ [
+ ("nonexistent_platform", "nonexistent_suite", KeyError),
+ ("unix", "nonexistent_suite", KeyError),
+ ("unix", "", TypeError),
+ ("", "", TypeError),
+ ("", "nonexistent_suite", TypeError),
+ ],
+)
+def test_get_runtimes_invalid(platform, suite, exception):
+ """Ensure get_runtimes() method raises an exception if improper request is made."""
+ with pytest.raises(exception):
+ chunking.get_runtimes(platform, suite)
+
+
+@pytest.mark.parametrize(
+ "suite",
+ [
+ "web-platform-tests",
+ "web-platform-tests-reftest",
+ "web-platform-tests-wdspec",
+ "web-platform-tests-crashtest",
+ ],
+)
+@pytest.mark.parametrize("chunks", [1, 3, 6, 20])
+def test_mock_chunk_manifests_wpt(unchunked_manifests, suite, chunks):
+ """Tests web-platform-tests and its subsuites chunking process."""
+ # Setup.
+ manifests = unchunked_manifests(suite)
+
+ # Generate the expected results, by generating list of indices that each
+ # manifest should go into and then appending each item to that index.
+ # This method is intentionally different from the way chunking.py performs
+ # chunking for cross-checking.
+ expected = [[] for _ in range(chunks)]
+ indexed = zip(manifests, list(range(0, chunks)) * len(manifests))
+ for i in indexed:
+ expected[i[1]].append(i[0])
+
+ # Call the method under test on unchunked manifests.
+ chunked_manifests = chunking.chunk_manifests(suite, "unix", chunks, manifests)
+
+ # Assertions and end test.
+ assert chunked_manifests
+ if chunks > len(manifests):
+ # If chunk count exceeds number of manifests, not all chunks will have
+ # manifests.
+ with pytest.raises(AssertionError):
+ assert all(chunked_manifests)
+ else:
+ assert all(chunked_manifests)
+ minimum = min(len(c) for c in chunked_manifests)
+ maximum = max(len(c) for c in chunked_manifests)
+ assert maximum - minimum <= 1
+ assert expected == chunked_manifests
+
+
+@pytest.mark.parametrize(
+ "suite",
+ [
+ "mochitest-devtools-chrome",
+ "mochitest-browser-chrome",
+ "mochitest-plain",
+ "mochitest-chrome",
+ "xpcshell",
+ ],
+)
+@pytest.mark.parametrize("chunks", [1, 3, 6, 20])
+def test_mock_chunk_manifests(
+ mock_manifest_runtimes, unchunked_manifests, suite, chunks
+):
+ """Tests non-WPT tests and its subsuites chunking process."""
+ # Setup.
+ manifests = unchunked_manifests(suite)
+
+ # Call the method under test on unchunked manifests.
+ chunked_manifests = chunking.chunk_manifests(suite, "unix", chunks, manifests)
+
+ # Assertions and end test.
+ assert chunked_manifests
+ if chunks > len(manifests):
+ # If chunk count exceeds number of manifests, not all chunks will have
+ # manifests.
+ with pytest.raises(AssertionError):
+ assert all(chunked_manifests)
+ else:
+ assert all(chunked_manifests)
+
+
+@pytest.mark.parametrize(
+ "suite",
+ [
+ "web-platform-tests",
+ "web-platform-tests-reftest",
+ "xpcshell",
+ "mochitest-plain",
+ "mochitest-devtools-chrome",
+ "mochitest-browser-chrome",
+ "mochitest-chrome",
+ ],
+)
+@pytest.mark.parametrize(
+ "platform",
+ [
+ ("mac", "x86_64"),
+ ("win", "x86_64"),
+ ("win", "x86"),
+ ("win", "aarch64"),
+ ("linux", "x86_64"),
+ ("linux", "x86"),
+ ],
+)
+def test_get_manifests(suite, platform, mock_mozinfo):
+ """Tests the DefaultLoader class' ability to load manifests."""
+ mozinfo = mock_mozinfo(*platform)
+
+ loader = chunking.DefaultLoader([])
+ manifests = loader.get_manifests(suite, frozenset(mozinfo.items()))
+
+ assert manifests
+ assert manifests["active"]
+ if "web-platform" in suite:
+ assert manifests["skipped"] == []
+ else:
+ assert manifests["skipped"]
+
+ items = manifests["active"]
+ if suite == "xpcshell":
+ assert all([re.search(r"xpcshell(.*)?.ini", m) for m in items])
+ if "mochitest" in suite:
+ assert all([re.search(r"(mochitest|chrome|browser).*.ini", m) for m in items])
+ if "web-platform" in suite:
+ assert all([m.startswith("/") and m.count("/") <= 4 for m in items])
+
+
+@pytest.mark.parametrize(
+ "suite",
+ [
+ "mochitest-devtools-chrome",
+ "mochitest-browser-chrome",
+ "mochitest-plain",
+ "mochitest-chrome",
+ "web-platform-tests",
+ "web-platform-tests-reftest",
+ "xpcshell",
+ ],
+)
+@pytest.mark.parametrize(
+ "platform",
+ [
+ ("mac", "x86_64"),
+ ("win", "x86_64"),
+ ("linux", "x86_64"),
+ ],
+)
+@pytest.mark.parametrize("chunks", [1, 3, 6, 20])
+def test_chunk_manifests(suite, platform, chunks, mock_mozinfo):
+ """Tests chunking with real manifests."""
+ mozinfo = mock_mozinfo(*platform)
+
+ loader = chunking.DefaultLoader([])
+ manifests = loader.get_manifests(suite, frozenset(mozinfo.items()))
+
+ chunked_manifests = chunking.chunk_manifests(
+ suite, platform, chunks, manifests["active"]
+ )
+
+ # Assertions and end test.
+ assert chunked_manifests
+ assert len(chunked_manifests) == chunks
+ assert all(chunked_manifests)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/taskcluster/gecko_taskgraph/test/test_util_docker.py b/taskcluster/gecko_taskgraph/test/test_util_docker.py
new file mode 100644
index 0000000000..49c01738fe
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/test_util_docker.py
@@ -0,0 +1,255 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import os
+import shutil
+import stat
+import tarfile
+import tempfile
+import unittest
+from unittest import mock
+
+import taskcluster_urls as liburls
+from mozunit import MockedOpen, main
+
+from gecko_taskgraph.util import docker
+
+MODE_STANDARD = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
+
+
+@mock.patch.dict("os.environ", {"TASKCLUSTER_ROOT_URL": liburls.test_root_url()})
+class TestDocker(unittest.TestCase):
+ def test_generate_context_hash(self):
+ tmpdir = tempfile.mkdtemp()
+ try:
+ os.makedirs(os.path.join(tmpdir, "docker", "my-image"))
+ p = os.path.join(tmpdir, "docker", "my-image", "Dockerfile")
+ with open(p, "w") as f:
+ f.write("FROM node\nADD a-file\n")
+ os.chmod(p, MODE_STANDARD)
+ p = os.path.join(tmpdir, "docker", "my-image", "a-file")
+ with open(p, "w") as f:
+ f.write("data\n")
+ os.chmod(p, MODE_STANDARD)
+ self.assertIn(
+ docker.generate_context_hash(
+ tmpdir,
+ os.path.join(tmpdir, "docker/my-image"),
+ "my-image",
+ {},
+ ),
+ (
+ "680532a33c845e3b4f8ea8a7bd697da579b647f28c29f7a0a71e51e6cca33983",
+ "cc02f943ae87b283749369fa9c4f6a74639c27a7b9972c99de58e5d9fb3a98ae",
+ ),
+ )
+ finally:
+ shutil.rmtree(tmpdir)
+
+ def test_docker_image_explicit_registry(self):
+ files = {}
+ files[f"{docker.IMAGE_DIR}/myimage/REGISTRY"] = "cool-images"
+ files[f"{docker.IMAGE_DIR}/myimage/VERSION"] = "1.2.3"
+ files[f"{docker.IMAGE_DIR}/myimage/HASH"] = "sha256:434..."
+ with MockedOpen(files):
+ self.assertEqual(
+ docker.docker_image("myimage"), "cool-images/myimage@sha256:434..."
+ )
+
+ def test_docker_image_explicit_registry_by_tag(self):
+ files = {}
+ files[f"{docker.IMAGE_DIR}/myimage/REGISTRY"] = "myreg"
+ files[f"{docker.IMAGE_DIR}/myimage/VERSION"] = "1.2.3"
+ files[f"{docker.IMAGE_DIR}/myimage/HASH"] = "sha256:434..."
+ with MockedOpen(files):
+ self.assertEqual(
+ docker.docker_image("myimage", by_tag=True), "myreg/myimage:1.2.3"
+ )
+
+ def test_docker_image_default_registry(self):
+ files = {}
+ files[f"{docker.IMAGE_DIR}/REGISTRY"] = "mozilla"
+ files[f"{docker.IMAGE_DIR}/myimage/VERSION"] = "1.2.3"
+ files[f"{docker.IMAGE_DIR}/myimage/HASH"] = "sha256:434..."
+ with MockedOpen(files):
+ self.assertEqual(
+ docker.docker_image("myimage"), "mozilla/myimage@sha256:434..."
+ )
+
+ def test_docker_image_default_registry_by_tag(self):
+ files = {}
+ files[f"{docker.IMAGE_DIR}/REGISTRY"] = "mozilla"
+ files[f"{docker.IMAGE_DIR}/myimage/VERSION"] = "1.2.3"
+ files[f"{docker.IMAGE_DIR}/myimage/HASH"] = "sha256:434..."
+ with MockedOpen(files):
+ self.assertEqual(
+ docker.docker_image("myimage", by_tag=True), "mozilla/myimage:1.2.3"
+ )
+
+ def test_create_context_tar_basic(self):
+ tmp = tempfile.mkdtemp()
+ try:
+ d = os.path.join(tmp, "test_image")
+ os.mkdir(d)
+ with open(os.path.join(d, "Dockerfile"), "a"):
+ pass
+ os.chmod(os.path.join(d, "Dockerfile"), MODE_STANDARD)
+
+ with open(os.path.join(d, "extra"), "a"):
+ pass
+ os.chmod(os.path.join(d, "extra"), MODE_STANDARD)
+
+ tp = os.path.join(tmp, "tar")
+ h = docker.create_context_tar(tmp, d, tp, "my_image", {})
+ self.assertIn(
+ h,
+ (
+ "eae3ad00936085eb3e5958912f79fb06ee8e14a91f7157c5f38625f7ddacb9c7",
+ "9ff54ee091c4f346e94e809b03efae5aa49a5c1db152f9f633682cfa005f7422",
+ ),
+ )
+
+ # File prefix should be "my_image"
+ with tarfile.open(tp, "r:gz") as tf:
+ self.assertEqual(
+ tf.getnames(),
+ [
+ "Dockerfile",
+ "extra",
+ ],
+ )
+ finally:
+ shutil.rmtree(tmp)
+
+ def test_create_context_topsrcdir_files(self):
+ tmp = tempfile.mkdtemp()
+ try:
+ d = os.path.join(tmp, "test-image")
+ os.mkdir(d)
+ with open(os.path.join(d, "Dockerfile"), "wb") as fh:
+ fh.write(b"# %include extra/file0\n")
+ os.chmod(os.path.join(d, "Dockerfile"), MODE_STANDARD)
+
+ extra = os.path.join(tmp, "extra")
+ os.mkdir(extra)
+ with open(os.path.join(extra, "file0"), "a"):
+ pass
+ os.chmod(os.path.join(extra, "file0"), MODE_STANDARD)
+
+ tp = os.path.join(tmp, "tar")
+ h = docker.create_context_tar(tmp, d, tp, "test_image", {})
+ self.assertIn(
+ h,
+ (
+ "49dc3827530cd344d7bcc52e1fdd4aefc632568cf442cffd3dd9633a58f271bf",
+ "8f8e3dd2b712003cd12bb39e5a84fc2a7c06e891cf481613a52bf3db472c4ca9",
+ ),
+ )
+
+ with tarfile.open(tp, "r:gz") as tf:
+ self.assertEqual(
+ tf.getnames(),
+ [
+ "Dockerfile",
+ "topsrcdir/extra/file0",
+ ],
+ )
+ finally:
+ shutil.rmtree(tmp)
+
+ def test_create_context_absolute_path(self):
+ tmp = tempfile.mkdtemp()
+ try:
+ d = os.path.join(tmp, "test-image")
+ os.mkdir(d)
+
+ # Absolute paths in %include syntax are not allowed.
+ with open(os.path.join(d, "Dockerfile"), "wb") as fh:
+ fh.write(b"# %include /etc/shadow\n")
+
+ with self.assertRaisesRegexp(Exception, "cannot be absolute"):
+ docker.create_context_tar(tmp, d, os.path.join(tmp, "tar"), "test", {})
+ finally:
+ shutil.rmtree(tmp)
+
+ def test_create_context_outside_topsrcdir(self):
+ tmp = tempfile.mkdtemp()
+ try:
+ d = os.path.join(tmp, "test-image")
+ os.mkdir(d)
+
+ with open(os.path.join(d, "Dockerfile"), "wb") as fh:
+ fh.write(b"# %include foo/../../../etc/shadow\n")
+
+ with self.assertRaisesRegexp(Exception, "path outside topsrcdir"):
+ docker.create_context_tar(tmp, d, os.path.join(tmp, "tar"), "test", {})
+ finally:
+ shutil.rmtree(tmp)
+
+ def test_create_context_missing_extra(self):
+ tmp = tempfile.mkdtemp()
+ try:
+ d = os.path.join(tmp, "test-image")
+ os.mkdir(d)
+
+ with open(os.path.join(d, "Dockerfile"), "wb") as fh:
+ fh.write(b"# %include does/not/exist\n")
+
+ with self.assertRaisesRegexp(Exception, "path does not exist"):
+ docker.create_context_tar(tmp, d, os.path.join(tmp, "tar"), "test", {})
+ finally:
+ shutil.rmtree(tmp)
+
+ def test_create_context_extra_directory(self):
+ tmp = tempfile.mkdtemp()
+ try:
+ d = os.path.join(tmp, "test-image")
+ os.mkdir(d)
+
+ with open(os.path.join(d, "Dockerfile"), "wb") as fh:
+ fh.write(b"# %include extra\n")
+ fh.write(b"# %include file0\n")
+ os.chmod(os.path.join(d, "Dockerfile"), MODE_STANDARD)
+
+ extra = os.path.join(tmp, "extra")
+ os.mkdir(extra)
+ for i in range(3):
+ p = os.path.join(extra, "file%d" % i)
+ with open(p, "wb") as fh:
+ fh.write(b"file%d" % i)
+ os.chmod(p, MODE_STANDARD)
+
+ with open(os.path.join(tmp, "file0"), "a"):
+ pass
+ os.chmod(os.path.join(tmp, "file0"), MODE_STANDARD)
+
+ tp = os.path.join(tmp, "tar")
+ h = docker.create_context_tar(tmp, d, tp, "my_image", {})
+
+ self.assertIn(
+ h,
+ (
+ "a392f23cd6606ae43116390a4d0113354cff1e688a41d46f48b0fb25e90baa13",
+ "02325bdc508c2e941959170beeb840f6bb91d0675cb8095783a7db7301d136b2",
+ ),
+ )
+
+ with tarfile.open(tp, "r:gz") as tf:
+ self.assertEqual(
+ tf.getnames(),
+ [
+ "Dockerfile",
+ "topsrcdir/extra/file0",
+ "topsrcdir/extra/file1",
+ "topsrcdir/extra/file2",
+ "topsrcdir/file0",
+ ],
+ )
+ finally:
+ shutil.rmtree(tmp)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/taskcluster/gecko_taskgraph/test/test_util_partials.py b/taskcluster/gecko_taskgraph/test/test_util_partials.py
new file mode 100644
index 0000000000..3630d7b0ec
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/test_util_partials.py
@@ -0,0 +1,128 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+from unittest import mock
+
+from mozunit import main
+
+from gecko_taskgraph.util import partials
+
+release_blob = {
+ "fileUrls": {
+ "release-localtest": {
+ "completes": {
+ "*": "%OS_FTP%/%LOCALE%/firefox-92.0.1.complete.mar",
+ }
+ }
+ },
+ "platforms": {
+ "WINNT_x86_64-msvc": {
+ "locales": {
+ "en-US": {
+ "buildID": "20210922161155",
+ }
+ }
+ }
+ },
+}
+
+
+def nightly_blob(release):
+ return {
+ "platforms": {
+ "WINNT_x86_64-msvc": {
+ "locales": {
+ "en-US": {
+ "buildID": release[-14:],
+ "completes": [{"fileUrl": release}],
+ }
+ }
+ }
+ }
+ }
+
+
+class TestReleaseHistory(unittest.TestCase):
+ @mock.patch("gecko_taskgraph.util.partials.get_release_builds")
+ @mock.patch("gecko_taskgraph.util.partials.get_sorted_releases")
+ def test_populate_release_history(self, get_sorted_releases, get_release_builds):
+ self.assertEqual(
+ partials.populate_release_history(
+ "Firefox", "mozilla-release", partial_updates={}
+ ),
+ {},
+ )
+ get_sorted_releases.assert_not_called()
+ get_release_builds.assert_not_called()
+
+ def patched_get_sorted_releases(product, branch):
+ assert branch == "mozilla-central"
+ return [
+ "Firefox-mozilla-central-nightly-20211003201113",
+ "Firefox-mozilla-central-nightly-20211003100640",
+ "Firefox-mozilla-central-nightly-20211002213629",
+ "Firefox-mozilla-central-nightly-20211002095048",
+ "Firefox-mozilla-central-nightly-20211001214601",
+ "Firefox-mozilla-central-nightly-20211001093323",
+ ]
+
+ def patched_get_release_builds(release, branch):
+ if branch == "mozilla-central":
+ return nightly_blob(release)
+ if branch == "mozilla-release":
+ return release_blob
+
+ get_sorted_releases.side_effect = patched_get_sorted_releases
+ get_release_builds.side_effect = patched_get_release_builds
+
+ self.assertEqual(
+ partials.populate_release_history(
+ "Firefox",
+ "mozilla-release",
+ partial_updates={"92.0.1": {"buildNumber": 1}},
+ ),
+ {
+ "WINNT_x86_64-msvc": {
+ "en-US": {
+ "target-92.0.1.partial.mar": {
+ "buildid": "20210922161155",
+ "mar_url": "win64/en-US/firefox-92.0.1.complete.mar",
+ "previousVersion": "92.0.1",
+ "previousBuildNumber": 1,
+ "product": "Firefox",
+ }
+ }
+ }
+ },
+ )
+ self.assertEqual(
+ partials.populate_release_history("Firefox", "mozilla-central"),
+ {
+ "WINNT_x86_64-msvc": {
+ "en-US": {
+ "target.partial-1.mar": {
+ "buildid": "20211003201113",
+ "mar_url": "Firefox-mozilla-central-nightly-20211003201113",
+ },
+ "target.partial-2.mar": {
+ "buildid": "20211003100640",
+ "mar_url": "Firefox-mozilla-central-nightly-20211003100640",
+ },
+ "target.partial-3.mar": {
+ "buildid": "20211002213629",
+ "mar_url": "Firefox-mozilla-central-nightly-20211002213629",
+ },
+ "target.partial-4.mar": {
+ "buildid": "20211002095048",
+ "mar_url": "Firefox-mozilla-central-nightly-20211002095048",
+ },
+ }
+ }
+ },
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/taskcluster/gecko_taskgraph/test/test_util_runnable_jobs.py b/taskcluster/gecko_taskgraph/test/test_util_runnable_jobs.py
new file mode 100644
index 0000000000..993521b830
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/test_util_runnable_jobs.py
@@ -0,0 +1,76 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import unittest
+
+from mozunit import main
+from taskgraph.graph import Graph
+from taskgraph.task import Task
+from taskgraph.taskgraph import TaskGraph
+
+from gecko_taskgraph.decision import full_task_graph_to_runnable_jobs
+
+
+class TestRunnableJobs(unittest.TestCase):
+
+ tasks = [
+ {
+ "kind": "build",
+ "label": "a",
+ "attributes": {},
+ "task": {
+ "extra": {"treeherder": {"symbol": "B"}},
+ },
+ },
+ {
+ "kind": "test",
+ "label": "b",
+ "attributes": {},
+ "task": {
+ "extra": {
+ "treeherder": {
+ "collection": {"opt": True},
+ "groupName": "Some group",
+ "groupSymbol": "GS",
+ "machine": {"platform": "linux64"},
+ "symbol": "t",
+ }
+ },
+ },
+ },
+ ]
+
+ def make_taskgraph(self, tasks):
+ label_to_taskid = {k: k + "-tid" for k in tasks}
+ for label, task_id in label_to_taskid.items():
+ tasks[label].task_id = task_id
+ graph = Graph(nodes=set(tasks), edges=set())
+ taskgraph = TaskGraph(tasks, graph)
+ return taskgraph, label_to_taskid
+
+ def test_taskgraph_to_runnable_jobs(self):
+ tg, label_to_taskid = self.make_taskgraph(
+ {t["label"]: Task(**t) for t in self.tasks[:]}
+ )
+
+ res = full_task_graph_to_runnable_jobs(tg.to_json())
+
+ self.assertEqual(
+ res,
+ {
+ "a": {"symbol": "B"},
+ "b": {
+ "collection": {"opt": True},
+ "groupName": "Some group",
+ "groupSymbol": "GS",
+ "symbol": "t",
+ "platform": "linux64",
+ },
+ },
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/taskcluster/gecko_taskgraph/test/test_util_templates.py b/taskcluster/gecko_taskgraph/test/test_util_templates.py
new file mode 100644
index 0000000000..edfb13a277
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/test_util_templates.py
@@ -0,0 +1,79 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import unittest
+
+import mozunit
+
+from gecko_taskgraph.util.templates import merge, merge_to
+
+
+class MergeTest(unittest.TestCase):
+ def test_merge_to_dicts(self):
+ source = {"a": 1, "b": 2}
+ dest = {"b": "20", "c": 30}
+ expected = {
+ "a": 1, # source only
+ "b": 2, # source overrides dest
+ "c": 30, # dest only
+ }
+ self.assertEqual(merge_to(source, dest), expected)
+ self.assertEqual(dest, expected)
+
+ def test_merge_to_lists(self):
+ source = {"x": [3, 4]}
+ dest = {"x": [1, 2]}
+ expected = {"x": [1, 2, 3, 4]} # dest first
+ self.assertEqual(merge_to(source, dest), expected)
+ self.assertEqual(dest, expected)
+
+ def test_merge_diff_types(self):
+ source = {"x": [1, 2]}
+ dest = {"x": "abc"}
+ expected = {"x": [1, 2]} # source wins
+ self.assertEqual(merge_to(source, dest), expected)
+ self.assertEqual(dest, expected)
+
+ def test_merge(self):
+ first = {"a": 1, "b": 2, "d": 11}
+ second = {"b": 20, "c": 30}
+ third = {"c": 300, "d": 400}
+ expected = {
+ "a": 1,
+ "b": 20,
+ "c": 300,
+ "d": 400,
+ }
+ self.assertEqual(merge(first, second, third), expected)
+
+ # inputs haven't changed..
+ self.assertEqual(first, {"a": 1, "b": 2, "d": 11})
+ self.assertEqual(second, {"b": 20, "c": 30})
+ self.assertEqual(third, {"c": 300, "d": 400})
+
+ def test_merge_by(self):
+ source = {
+ "x": "abc",
+ "y": {"by-foo": {"quick": "fox", "default": ["a", "b", "c"]}},
+ }
+ dest = {"y": {"by-foo": {"purple": "rain", "default": ["x", "y", "z"]}}}
+ expected = {
+ "x": "abc",
+ "y": {"by-foo": {"quick": "fox", "default": ["a", "b", "c"]}},
+ } # source wins
+ self.assertEqual(merge_to(source, dest), expected)
+ self.assertEqual(dest, expected)
+
+ def test_merge_multiple_by(self):
+ source = {"x": {"by-foo": {"quick": "fox", "default": ["a", "b", "c"]}}}
+ dest = {"x": {"by-bar": {"purple": "rain", "default": ["x", "y", "z"]}}}
+ expected = {
+ "x": {"by-foo": {"quick": "fox", "default": ["a", "b", "c"]}}
+ } # source wins
+ self.assertEqual(merge_to(source, dest), expected)
+
+
+if __name__ == "__main__":
+ mozunit.main()
diff --git a/taskcluster/gecko_taskgraph/test/test_util_verify.py b/taskcluster/gecko_taskgraph/test/test_util_verify.py
new file mode 100644
index 0000000000..e2f774a315
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/test_util_verify.py
@@ -0,0 +1,149 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+There are some basic tests run as part of the Decision task to make sure
+documentation exists for taskgraph functionality.
+These functions are defined in gecko_taskgraph.generator and call
+gecko_taskgraph.util.verify.verify_docs with different parameters to do the
+actual checking.
+"""
+
+
+import os.path
+
+import pytest
+from mozunit import main
+
+import gecko_taskgraph.util.verify
+from gecko_taskgraph import GECKO
+from gecko_taskgraph.util.verify import DocPaths, verify_docs
+
+FF_DOCS_BASE = os.path.join(GECKO, "taskcluster", "docs")
+EXTRA_DOCS_BASE = os.path.abspath(os.path.join(os.path.dirname(__file__), "docs"))
+
+
+@pytest.fixture
+def mock_single_doc_path(monkeypatch):
+ """Set a single path containing documentation"""
+ mocked_documentation_paths = DocPaths()
+ mocked_documentation_paths.add(FF_DOCS_BASE)
+ monkeypatch.setattr(
+ gecko_taskgraph.util.verify, "documentation_paths", mocked_documentation_paths
+ )
+
+
+@pytest.fixture
+def mock_two_doc_paths(monkeypatch):
+ """Set two paths containing documentation"""
+ mocked_documentation_paths = DocPaths()
+ mocked_documentation_paths.add(FF_DOCS_BASE)
+ mocked_documentation_paths.add(EXTRA_DOCS_BASE)
+ monkeypatch.setattr(
+ gecko_taskgraph.util.verify, "documentation_paths", mocked_documentation_paths
+ )
+
+
+@pytest.mark.usefixtures("mock_single_doc_path")
+class PyTestSingleDocPath:
+ """
+ Taskcluster documentation for Firefox is in a single directory. Check the tests
+ running at build time to make sure documentation exists, actually work themselves.
+ """
+
+ def test_heading(self):
+ """
+ Look for a headings in filename matching identifiers. This is used when making sure
+ documentation exists for kinds and attributes.
+ """
+ verify_docs(
+ filename="kinds.rst",
+ identifiers=["build", "packages", "toolchain"],
+ appearing_as="heading",
+ )
+ with pytest.raises(Exception, match="missing from doc file"):
+ verify_docs(
+ filename="kinds.rst",
+ identifiers=["build", "packages", "badvalue"],
+ appearing_as="heading",
+ )
+
+ def test_inline_literal(self):
+ """
+ Look for inline-literals in filename. Used when checking documentation for decision
+ task parameters and run-using functions.
+ """
+ verify_docs(
+ filename="parameters.rst",
+ identifiers=["base_repository", "head_repository", "owner"],
+ appearing_as="inline-literal",
+ )
+ with pytest.raises(Exception, match="missing from doc file"):
+ verify_docs(
+ filename="parameters.rst",
+ identifiers=["base_repository", "head_repository", "badvalue"],
+ appearing_as="inline-literal",
+ )
+
+
+@pytest.mark.usefixtures("mock_two_doc_paths")
+class PyTestTwoDocPaths:
+ """
+ Thunderbird extends Firefox's taskgraph with additional kinds. The documentation
+ for Thunderbird kinds are in its repository, and documentation_paths will have
+ two places to look for files. Run the same tests as for a single documentation
+ path, and cover additional possible scenarios.
+ """
+
+ def test_heading(self):
+ """
+ Look for a headings in filename matching identifiers. This is used when
+ making sure documentation exists for kinds and attributes.
+ The first test looks for headings that are all within the first doc path,
+ the second test is new and has a heading found in the second path.
+ The final check has a identifier that will not match and should
+ produce an error.
+ """
+ verify_docs(
+ filename="kinds.rst",
+ identifiers=["build", "packages", "toolchain"],
+ appearing_as="heading",
+ )
+ verify_docs(
+ filename="kinds.rst",
+ identifiers=["build", "packages", "newkind"],
+ appearing_as="heading",
+ )
+ with pytest.raises(Exception, match="missing from doc file"):
+ verify_docs(
+ filename="kinds.rst",
+ identifiers=["build", "packages", "badvalue"],
+ appearing_as="heading",
+ )
+
+ def test_inline_literal(self):
+ """
+ Look for inline-literals in filename. Used when checking documentation for decision
+ task parameters and run-using functions. As with the heading tests,
+ the second check looks for an identifier in the added documentation path.
+ """
+ verify_docs(
+ filename="parameters.rst",
+ identifiers=["base_repository", "head_repository", "owner"],
+ appearing_as="inline-literal",
+ )
+ verify_docs(
+ filename="parameters.rst",
+ identifiers=["base_repository", "head_repository", "newparameter"],
+ appearing_as="inline-literal",
+ )
+ with pytest.raises(Exception, match="missing from doc file"):
+ verify_docs(
+ filename="parameters.rst",
+ identifiers=["base_repository", "head_repository", "badvalue"],
+ appearing_as="inline-literal",
+ )
+
+
+if __name__ == "__main__":
+ main()
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-25 18:51:31 +0000