diff options
Diffstat (limited to 'taskcluster/gecko_taskgraph/test/test_try_option_syntax.py')
| -rw-r--r-- | taskcluster/gecko_taskgraph/test/test_try_option_syntax.py | 430 |
1 files changed, 430 insertions, 0 deletions
diff --git a/taskcluster/gecko_taskgraph/test/test_try_option_syntax.py b/taskcluster/gecko_taskgraph/test/test_try_option_syntax.py new file mode 100644 index 0000000000..a37de53378 --- /dev/null +++ b/taskcluster/gecko_taskgraph/test/test_try_option_syntax.py @@ -0,0 +1,430 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import unittest + +from mozunit import main +from taskgraph.graph import Graph +from taskgraph.task import Task +from taskgraph.taskgraph import TaskGraph + +from gecko_taskgraph.try_option_syntax import TryOptionSyntax, parse_message + + +def unittest_task(n, tp, bt="opt"): + return ( + n, + Task( + "test", + n, + { + "unittest_try_name": n, + "test_platform": tp.split("/")[0], + "build_type": bt, + }, + {}, + ), + ) + + +def talos_task(n, tp, bt="opt"): + return ( + n, + Task( + "test", + n, + { + "talos_try_name": n, + "test_platform": tp.split("/")[0], + "build_type": bt, + }, + {}, + ), + ) + + +tasks = { + k: v + for k, v in [ + unittest_task("mochitest-browser-chrome", "linux/opt"), + unittest_task("mochitest-browser-chrome-e10s", "linux64/opt"), + unittest_task("mochitest-chrome", "linux/debug", "debug"), + unittest_task("mochitest-webgl1-core", "linux/debug", "debug"), + unittest_task("mochitest-webgl1-ext", "linux/debug", "debug"), + unittest_task("mochitest-webgl2-core", "linux/debug", "debug"), + unittest_task("mochitest-webgl2-ext", "linux/debug", "debug"), + unittest_task("mochitest-webgl2-deqp", "linux/debug", "debug"), + unittest_task("extra1", "linux", "debug/opt"), + unittest_task("extra2", "win32/opt"), + unittest_task("crashtest-e10s", "linux/other"), + unittest_task("gtest", "linux64/asan"), + talos_task("dromaeojs", "linux64/psan"), + unittest_task("extra3", "linux/opt"), + unittest_task("extra4", "linux64/debug", "debug"), + unittest_task("extra5", "linux/this"), + unittest_task("extra6", "linux/that"), + unittest_task("extra7", "linux/other"), + unittest_task("extra8", "linux64/asan"), + talos_task("extra9", "linux64/psan"), + ] +} + +RIDEALONG_BUILDS = { + "linux": ["linux-ridealong"], + "linux64": ["linux64-ridealong"], +} + +GRAPH_CONFIG = { + "try": {"ridealong-builds": RIDEALONG_BUILDS}, +} + +for r in RIDEALONG_BUILDS.values(): + tasks.update({k: v for k, v in [unittest_task(n + "-test", n) for n in r]}) + +unittest_tasks = {k: v for k, v in tasks.items() if "unittest_try_name" in v.attributes} +talos_tasks = {k: v for k, v in tasks.items() if "talos_try_name" in v.attributes} +graph_with_jobs = TaskGraph(tasks, Graph(set(tasks), set())) + + +class TestTryOptionSyntax(unittest.TestCase): + def test_unknown_args(self): + "unknown arguments are ignored" + parameters = parse_message("try: --doubledash -z extra") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + # equilvant to "try:".. + self.assertEqual(tos.build_types, []) + self.assertEqual(tos.jobs, []) + + def test_apostrophe_in_message(self): + "apostrophe does not break parsing" + parameters = parse_message("Increase spammy log's log level. try: -b do") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(sorted(tos.build_types), ["debug", "opt"]) + + def test_b_do(self): + "-b do should produce both build_types" + parameters = parse_message("try: -b do") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(sorted(tos.build_types), ["debug", "opt"]) + + def test_b_d(self): + "-b d should produce build_types=['debug']" + parameters = parse_message("try: -b d") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(sorted(tos.build_types), ["debug"]) + + def test_b_o(self): + "-b o should produce build_types=['opt']" + parameters = parse_message("try: -b o") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(sorted(tos.build_types), ["opt"]) + + def test_build_o(self): + "--build o should produce build_types=['opt']" + parameters = parse_message("try: --build o") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(sorted(tos.build_types), ["opt"]) + + def test_b_dx(self): + "-b dx should produce build_types=['debug'], silently ignoring the x" + parameters = parse_message("try: -b dx") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(sorted(tos.build_types), ["debug"]) + + def test_j_job(self): + "-j somejob sets jobs=['somejob']" + parameters = parse_message("try: -j somejob") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(sorted(tos.jobs), ["somejob"]) + + def test_j_jobs(self): + "-j job1,job2 sets jobs=['job1', 'job2']" + parameters = parse_message("try: -j job1,job2") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(sorted(tos.jobs), ["job1", "job2"]) + + def test_j_all(self): + "-j all sets jobs=None" + parameters = parse_message("try: -j all") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.jobs, None) + + def test_j_twice(self): + "-j job1 -j job2 sets jobs=job1, job2" + parameters = parse_message("try: -j job1 -j job2") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(sorted(tos.jobs), sorted(["job1", "job2"])) + + def test_p_all(self): + "-p all sets platforms=None" + parameters = parse_message("try: -p all") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.platforms, None) + + def test_p_linux(self): + "-p linux sets platforms=['linux', 'linux-ridealong']" + parameters = parse_message("try: -p linux") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.platforms, ["linux", "linux-ridealong"]) + + def test_p_linux_win32(self): + "-p linux,win32 sets platforms=['linux', 'linux-ridealong', 'win32']" + parameters = parse_message("try: -p linux,win32") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(sorted(tos.platforms), ["linux", "linux-ridealong", "win32"]) + + def test_p_expands_ridealongs(self): + "-p linux,linux64 includes the RIDEALONG_BUILDS" + parameters = parse_message("try: -p linux,linux64") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + platforms = set(["linux"] + RIDEALONG_BUILDS["linux"]) + platforms |= set(["linux64"] + RIDEALONG_BUILDS["linux64"]) + self.assertEqual(sorted(tos.platforms), sorted(platforms)) + + def test_u_none(self): + "-u none sets unittests=[]" + parameters = parse_message("try: -u none") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.unittests, []) + + def test_u_all(self): + "-u all sets unittests=[..whole list..]" + parameters = parse_message("try: -u all") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.unittests, [{"test": t} for t in sorted(unittest_tasks)]) + + def test_u_single(self): + "-u mochitest-webgl1-core sets unittests=[mochitest-webgl1-core]" + parameters = parse_message("try: -u mochitest-webgl1-core") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.unittests, [{"test": "mochitest-webgl1-core"}]) + + def test_u_alias(self): + "-u mochitest-gl sets unittests=[mochitest-webgl*]" + parameters = parse_message("try: -u mochitest-gl") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual( + tos.unittests, + [ + {"test": t} + for t in [ + "mochitest-webgl1-core", + "mochitest-webgl1-ext", + "mochitest-webgl2-core", + "mochitest-webgl2-deqp", + "mochitest-webgl2-ext", + ] + ], + ) + + def test_u_multi_alias(self): + "-u e10s sets unittests=[all e10s unittests]" + parameters = parse_message("try: -u e10s") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual( + tos.unittests, [{"test": t} for t in sorted(unittest_tasks) if "e10s" in t] + ) + + def test_u_commas(self): + "-u mochitest-webgl1-core,gtest sets unittests=both" + parameters = parse_message("try: -u mochitest-webgl1-core,gtest") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual( + tos.unittests, + [ + {"test": "gtest"}, + {"test": "mochitest-webgl1-core"}, + ], + ) + + def test_u_chunks(self): + "-u gtest-3,gtest-4 selects the third and fourth chunk of gtest" + parameters = parse_message("try: -u gtest-3,gtest-4") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual( + sorted(tos.unittests), + sorted( + [ + {"test": "gtest", "only_chunks": set("34")}, + ] + ), + ) + + def test_u_platform(self): + "-u gtest[linux] selects the linux platform for gtest" + parameters = parse_message("try: -u gtest[linux]") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual( + sorted(tos.unittests), + sorted( + [ + {"test": "gtest", "platforms": ["linux"]}, + ] + ), + ) + + def test_u_platforms(self): + "-u gtest[linux,win32] selects the linux and win32 platforms for gtest" + parameters = parse_message("try: -u gtest[linux,win32]") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual( + sorted(tos.unittests), + sorted( + [ + {"test": "gtest", "platforms": ["linux", "win32"]}, + ] + ), + ) + + def test_u_platforms_pretty(self): + """-u gtest[Ubuntu] selects the linux, linux64 and linux64-asan + platforms for gtest""" + parameters = parse_message("try: -u gtest[Ubuntu]") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual( + sorted(tos.unittests), + sorted( + [ + { + "test": "gtest", + "platforms": [ + "linux32", + "linux64", + "linux64-asan", + "linux1804-64", + "linux1804-64-asan", + ], + }, + ] + ), + ) + + def test_u_platforms_negated(self): + "-u gtest[-linux] selects all platforms but linux for gtest" + parameters = parse_message("try: -u gtest[-linux]") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + all_platforms = {x.attributes["test_platform"] for x in unittest_tasks.values()} + self.assertEqual( + sorted(tos.unittests[0]["platforms"]), + sorted(x for x in all_platforms if x != "linux"), + ) + + def test_u_platforms_negated_pretty(self): + "-u gtest[Ubuntu,-x64] selects just linux for gtest" + parameters = parse_message("try: -u gtest[Ubuntu,-x64]") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual( + sorted(tos.unittests), + sorted( + [ + {"test": "gtest", "platforms": ["linux32"]}, + ] + ), + ) + + def test_u_chunks_platforms(self): + "-u gtest-1[linux,win32] selects the linux and win32 platforms for chunk 1 of gtest" + parameters = parse_message("try: -u gtest-1[linux,win32]") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual( + tos.unittests, + [ + { + "test": "gtest", + "platforms": ["linux", "win32"], + "only_chunks": set("1"), + }, + ], + ) + + def test_t_none(self): + "-t none sets talos=[]" + parameters = parse_message("try: -t none") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.talos, []) + + def test_t_all(self): + "-t all sets talos=[..whole list..]" + parameters = parse_message("try: -t all") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.talos, [{"test": t} for t in sorted(talos_tasks)]) + + def test_t_single(self): + "-t mochitest-webgl sets talos=[mochitest-webgl]" + parameters = parse_message("try: -t mochitest-webgl") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.talos, [{"test": "mochitest-webgl"}]) + + # -t shares an implementation with -u, so it's not tested heavily + + def test_trigger_tests(self): + "--rebuild 10 sets trigger_tests" + parameters = parse_message("try: --rebuild 10") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.trigger_tests, 10) + + def test_talos_trigger_tests(self): + "--rebuild-talos 10 sets talos_trigger_tests" + parameters = parse_message("try: --rebuild-talos 10") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.talos_trigger_tests, 10) + + def test_interactive(self): + "--interactive sets interactive" + parameters = parse_message("try: --interactive") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.interactive, True) + + def test_all_email(self): + "--all-emails sets notifications" + parameters = parse_message("try: --all-emails") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.notifications, "all") + + def test_fail_email(self): + "--failure-emails sets notifications" + parameters = parse_message("try: --failure-emails") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.notifications, "failure") + + def test_no_email(self): + "no email settings don't set notifications" + parameters = parse_message("try:") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.notifications, None) + + def test_setenv(self): + "--setenv VAR=value adds a environment variables setting to env" + parameters = parse_message("try: --setenv VAR1=value1 --setenv VAR2=value2") + assert parameters["try_task_config"]["env"] == { + "VAR1": "value1", + "VAR2": "value2", + } + + def test_profile(self): + "--gecko-profile sets profile to true" + parameters = parse_message("try: --gecko-profile") + assert parameters["try_task_config"]["gecko-profile"] is True + + def test_tag(self): + "--tag TAG sets tag to TAG value" + parameters = parse_message("try: --tag tagName") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertEqual(tos.tag, "tagName") + + def test_no_retry(self): + "--no-retry sets no_retry to true" + parameters = parse_message("try: --no-retry") + tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG) + self.assertTrue(tos.no_retry) + + def test_artifact(self): + "--artifact sets artifact to true" + parameters = parse_message("try: --artifact") + assert parameters["try_task_config"]["use-artifact-builds"] is True + + +if __name__ == "__main__": + main() |