summaryrefslogtreecommitdiffstats
path: root/taskcluster/gecko_taskgraph/test/test_try_option_syntax.py
diff options
context:
space:
mode:
Diffstat (limited to 'taskcluster/gecko_taskgraph/test/test_try_option_syntax.py')
-rw-r--r--taskcluster/gecko_taskgraph/test/test_try_option_syntax.py430
1 files changed, 430 insertions, 0 deletions
diff --git a/taskcluster/gecko_taskgraph/test/test_try_option_syntax.py b/taskcluster/gecko_taskgraph/test/test_try_option_syntax.py
new file mode 100644
index 0000000000..a37de53378
--- /dev/null
+++ b/taskcluster/gecko_taskgraph/test/test_try_option_syntax.py
@@ -0,0 +1,430 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import unittest
+
+from mozunit import main
+from taskgraph.graph import Graph
+from taskgraph.task import Task
+from taskgraph.taskgraph import TaskGraph
+
+from gecko_taskgraph.try_option_syntax import TryOptionSyntax, parse_message
+
+
+def unittest_task(n, tp, bt="opt"):
+ return (
+ n,
+ Task(
+ "test",
+ n,
+ {
+ "unittest_try_name": n,
+ "test_platform": tp.split("/")[0],
+ "build_type": bt,
+ },
+ {},
+ ),
+ )
+
+
+def talos_task(n, tp, bt="opt"):
+ return (
+ n,
+ Task(
+ "test",
+ n,
+ {
+ "talos_try_name": n,
+ "test_platform": tp.split("/")[0],
+ "build_type": bt,
+ },
+ {},
+ ),
+ )
+
+
+tasks = {
+ k: v
+ for k, v in [
+ unittest_task("mochitest-browser-chrome", "linux/opt"),
+ unittest_task("mochitest-browser-chrome-e10s", "linux64/opt"),
+ unittest_task("mochitest-chrome", "linux/debug", "debug"),
+ unittest_task("mochitest-webgl1-core", "linux/debug", "debug"),
+ unittest_task("mochitest-webgl1-ext", "linux/debug", "debug"),
+ unittest_task("mochitest-webgl2-core", "linux/debug", "debug"),
+ unittest_task("mochitest-webgl2-ext", "linux/debug", "debug"),
+ unittest_task("mochitest-webgl2-deqp", "linux/debug", "debug"),
+ unittest_task("extra1", "linux", "debug/opt"),
+ unittest_task("extra2", "win32/opt"),
+ unittest_task("crashtest-e10s", "linux/other"),
+ unittest_task("gtest", "linux64/asan"),
+ talos_task("dromaeojs", "linux64/psan"),
+ unittest_task("extra3", "linux/opt"),
+ unittest_task("extra4", "linux64/debug", "debug"),
+ unittest_task("extra5", "linux/this"),
+ unittest_task("extra6", "linux/that"),
+ unittest_task("extra7", "linux/other"),
+ unittest_task("extra8", "linux64/asan"),
+ talos_task("extra9", "linux64/psan"),
+ ]
+}
+
+RIDEALONG_BUILDS = {
+ "linux": ["linux-ridealong"],
+ "linux64": ["linux64-ridealong"],
+}
+
+GRAPH_CONFIG = {
+ "try": {"ridealong-builds": RIDEALONG_BUILDS},
+}
+
+for r in RIDEALONG_BUILDS.values():
+ tasks.update({k: v for k, v in [unittest_task(n + "-test", n) for n in r]})
+
+unittest_tasks = {k: v for k, v in tasks.items() if "unittest_try_name" in v.attributes}
+talos_tasks = {k: v for k, v in tasks.items() if "talos_try_name" in v.attributes}
+graph_with_jobs = TaskGraph(tasks, Graph(set(tasks), set()))
+
+
+class TestTryOptionSyntax(unittest.TestCase):
+ def test_unknown_args(self):
+ "unknown arguments are ignored"
+ parameters = parse_message("try: --doubledash -z extra")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ # equilvant to "try:"..
+ self.assertEqual(tos.build_types, [])
+ self.assertEqual(tos.jobs, [])
+
+ def test_apostrophe_in_message(self):
+ "apostrophe does not break parsing"
+ parameters = parse_message("Increase spammy log's log level. try: -b do")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(sorted(tos.build_types), ["debug", "opt"])
+
+ def test_b_do(self):
+ "-b do should produce both build_types"
+ parameters = parse_message("try: -b do")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(sorted(tos.build_types), ["debug", "opt"])
+
+ def test_b_d(self):
+ "-b d should produce build_types=['debug']"
+ parameters = parse_message("try: -b d")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(sorted(tos.build_types), ["debug"])
+
+ def test_b_o(self):
+ "-b o should produce build_types=['opt']"
+ parameters = parse_message("try: -b o")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(sorted(tos.build_types), ["opt"])
+
+ def test_build_o(self):
+ "--build o should produce build_types=['opt']"
+ parameters = parse_message("try: --build o")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(sorted(tos.build_types), ["opt"])
+
+ def test_b_dx(self):
+ "-b dx should produce build_types=['debug'], silently ignoring the x"
+ parameters = parse_message("try: -b dx")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(sorted(tos.build_types), ["debug"])
+
+ def test_j_job(self):
+ "-j somejob sets jobs=['somejob']"
+ parameters = parse_message("try: -j somejob")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(sorted(tos.jobs), ["somejob"])
+
+ def test_j_jobs(self):
+ "-j job1,job2 sets jobs=['job1', 'job2']"
+ parameters = parse_message("try: -j job1,job2")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(sorted(tos.jobs), ["job1", "job2"])
+
+ def test_j_all(self):
+ "-j all sets jobs=None"
+ parameters = parse_message("try: -j all")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.jobs, None)
+
+ def test_j_twice(self):
+ "-j job1 -j job2 sets jobs=job1, job2"
+ parameters = parse_message("try: -j job1 -j job2")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(sorted(tos.jobs), sorted(["job1", "job2"]))
+
+ def test_p_all(self):
+ "-p all sets platforms=None"
+ parameters = parse_message("try: -p all")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.platforms, None)
+
+ def test_p_linux(self):
+ "-p linux sets platforms=['linux', 'linux-ridealong']"
+ parameters = parse_message("try: -p linux")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.platforms, ["linux", "linux-ridealong"])
+
+ def test_p_linux_win32(self):
+ "-p linux,win32 sets platforms=['linux', 'linux-ridealong', 'win32']"
+ parameters = parse_message("try: -p linux,win32")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(sorted(tos.platforms), ["linux", "linux-ridealong", "win32"])
+
+ def test_p_expands_ridealongs(self):
+ "-p linux,linux64 includes the RIDEALONG_BUILDS"
+ parameters = parse_message("try: -p linux,linux64")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ platforms = set(["linux"] + RIDEALONG_BUILDS["linux"])
+ platforms |= set(["linux64"] + RIDEALONG_BUILDS["linux64"])
+ self.assertEqual(sorted(tos.platforms), sorted(platforms))
+
+ def test_u_none(self):
+ "-u none sets unittests=[]"
+ parameters = parse_message("try: -u none")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.unittests, [])
+
+ def test_u_all(self):
+ "-u all sets unittests=[..whole list..]"
+ parameters = parse_message("try: -u all")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.unittests, [{"test": t} for t in sorted(unittest_tasks)])
+
+ def test_u_single(self):
+ "-u mochitest-webgl1-core sets unittests=[mochitest-webgl1-core]"
+ parameters = parse_message("try: -u mochitest-webgl1-core")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.unittests, [{"test": "mochitest-webgl1-core"}])
+
+ def test_u_alias(self):
+ "-u mochitest-gl sets unittests=[mochitest-webgl*]"
+ parameters = parse_message("try: -u mochitest-gl")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(
+ tos.unittests,
+ [
+ {"test": t}
+ for t in [
+ "mochitest-webgl1-core",
+ "mochitest-webgl1-ext",
+ "mochitest-webgl2-core",
+ "mochitest-webgl2-deqp",
+ "mochitest-webgl2-ext",
+ ]
+ ],
+ )
+
+ def test_u_multi_alias(self):
+ "-u e10s sets unittests=[all e10s unittests]"
+ parameters = parse_message("try: -u e10s")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(
+ tos.unittests, [{"test": t} for t in sorted(unittest_tasks) if "e10s" in t]
+ )
+
+ def test_u_commas(self):
+ "-u mochitest-webgl1-core,gtest sets unittests=both"
+ parameters = parse_message("try: -u mochitest-webgl1-core,gtest")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(
+ tos.unittests,
+ [
+ {"test": "gtest"},
+ {"test": "mochitest-webgl1-core"},
+ ],
+ )
+
+ def test_u_chunks(self):
+ "-u gtest-3,gtest-4 selects the third and fourth chunk of gtest"
+ parameters = parse_message("try: -u gtest-3,gtest-4")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(
+ sorted(tos.unittests),
+ sorted(
+ [
+ {"test": "gtest", "only_chunks": set("34")},
+ ]
+ ),
+ )
+
+ def test_u_platform(self):
+ "-u gtest[linux] selects the linux platform for gtest"
+ parameters = parse_message("try: -u gtest[linux]")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(
+ sorted(tos.unittests),
+ sorted(
+ [
+ {"test": "gtest", "platforms": ["linux"]},
+ ]
+ ),
+ )
+
+ def test_u_platforms(self):
+ "-u gtest[linux,win32] selects the linux and win32 platforms for gtest"
+ parameters = parse_message("try: -u gtest[linux,win32]")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(
+ sorted(tos.unittests),
+ sorted(
+ [
+ {"test": "gtest", "platforms": ["linux", "win32"]},
+ ]
+ ),
+ )
+
+ def test_u_platforms_pretty(self):
+ """-u gtest[Ubuntu] selects the linux, linux64 and linux64-asan
+ platforms for gtest"""
+ parameters = parse_message("try: -u gtest[Ubuntu]")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(
+ sorted(tos.unittests),
+ sorted(
+ [
+ {
+ "test": "gtest",
+ "platforms": [
+ "linux32",
+ "linux64",
+ "linux64-asan",
+ "linux1804-64",
+ "linux1804-64-asan",
+ ],
+ },
+ ]
+ ),
+ )
+
+ def test_u_platforms_negated(self):
+ "-u gtest[-linux] selects all platforms but linux for gtest"
+ parameters = parse_message("try: -u gtest[-linux]")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ all_platforms = {x.attributes["test_platform"] for x in unittest_tasks.values()}
+ self.assertEqual(
+ sorted(tos.unittests[0]["platforms"]),
+ sorted(x for x in all_platforms if x != "linux"),
+ )
+
+ def test_u_platforms_negated_pretty(self):
+ "-u gtest[Ubuntu,-x64] selects just linux for gtest"
+ parameters = parse_message("try: -u gtest[Ubuntu,-x64]")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(
+ sorted(tos.unittests),
+ sorted(
+ [
+ {"test": "gtest", "platforms": ["linux32"]},
+ ]
+ ),
+ )
+
+ def test_u_chunks_platforms(self):
+ "-u gtest-1[linux,win32] selects the linux and win32 platforms for chunk 1 of gtest"
+ parameters = parse_message("try: -u gtest-1[linux,win32]")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(
+ tos.unittests,
+ [
+ {
+ "test": "gtest",
+ "platforms": ["linux", "win32"],
+ "only_chunks": set("1"),
+ },
+ ],
+ )
+
+ def test_t_none(self):
+ "-t none sets talos=[]"
+ parameters = parse_message("try: -t none")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.talos, [])
+
+ def test_t_all(self):
+ "-t all sets talos=[..whole list..]"
+ parameters = parse_message("try: -t all")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.talos, [{"test": t} for t in sorted(talos_tasks)])
+
+ def test_t_single(self):
+ "-t mochitest-webgl sets talos=[mochitest-webgl]"
+ parameters = parse_message("try: -t mochitest-webgl")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.talos, [{"test": "mochitest-webgl"}])
+
+ # -t shares an implementation with -u, so it's not tested heavily
+
+ def test_trigger_tests(self):
+ "--rebuild 10 sets trigger_tests"
+ parameters = parse_message("try: --rebuild 10")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.trigger_tests, 10)
+
+ def test_talos_trigger_tests(self):
+ "--rebuild-talos 10 sets talos_trigger_tests"
+ parameters = parse_message("try: --rebuild-talos 10")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.talos_trigger_tests, 10)
+
+ def test_interactive(self):
+ "--interactive sets interactive"
+ parameters = parse_message("try: --interactive")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.interactive, True)
+
+ def test_all_email(self):
+ "--all-emails sets notifications"
+ parameters = parse_message("try: --all-emails")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.notifications, "all")
+
+ def test_fail_email(self):
+ "--failure-emails sets notifications"
+ parameters = parse_message("try: --failure-emails")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.notifications, "failure")
+
+ def test_no_email(self):
+ "no email settings don't set notifications"
+ parameters = parse_message("try:")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.notifications, None)
+
+ def test_setenv(self):
+ "--setenv VAR=value adds a environment variables setting to env"
+ parameters = parse_message("try: --setenv VAR1=value1 --setenv VAR2=value2")
+ assert parameters["try_task_config"]["env"] == {
+ "VAR1": "value1",
+ "VAR2": "value2",
+ }
+
+ def test_profile(self):
+ "--gecko-profile sets profile to true"
+ parameters = parse_message("try: --gecko-profile")
+ assert parameters["try_task_config"]["gecko-profile"] is True
+
+ def test_tag(self):
+ "--tag TAG sets tag to TAG value"
+ parameters = parse_message("try: --tag tagName")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.tag, "tagName")
+
+ def test_no_retry(self):
+ "--no-retry sets no_retry to true"
+ parameters = parse_message("try: --no-retry")
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertTrue(tos.no_retry)
+
+ def test_artifact(self):
+ "--artifact sets artifact to true"
+ parameters = parse_message("try: --artifact")
+ assert parameters["try_task_config"]["use-artifact-builds"] is True
+
+
+if __name__ == "__main__":
+ main()