path: root/taskcluster/taskgraph/test/test_try_option_syntax.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import absolute_import, print_function, unicode_literals

import unittest

import six
from taskgraph.try_option_syntax import TryOptionSyntax, parse_message
from taskgraph.graph import Graph
from taskgraph.taskgraph import TaskGraph
from taskgraph.task import Task
from mozunit import main


def unittest_task(n, tp, bt="opt"):
    return (
        n,
        Task(
            "test",
            n,
            {
                "unittest_try_name": n,
                "test_platform": tp.split("/")[0],
                "build_type": bt,
            },
            {},
        ),
    )


def talos_task(n, tp, bt="opt"):
    return (
        n,
        Task(
            "test",
            n,
            {
                "talos_try_name": n,
                "test_platform": tp.split("/")[0],
                "build_type": bt,
            },
            {},
        ),
    )


tasks = {
    k: v
    for k, v in [
        unittest_task("mochitest-browser-chrome", "linux/opt"),
        unittest_task("mochitest-browser-chrome-e10s", "linux64/opt"),
        unittest_task("mochitest-chrome", "linux/debug", "debug"),
        unittest_task("mochitest-webgl1-core", "linux/debug", "debug"),
        unittest_task("mochitest-webgl1-ext", "linux/debug", "debug"),
        unittest_task("mochitest-webgl2-core", "linux/debug", "debug"),
        unittest_task("mochitest-webgl2-ext", "linux/debug", "debug"),
        unittest_task("mochitest-webgl2-deqp", "linux/debug", "debug"),
        unittest_task("extra1", "linux", "debug/opt"),
        unittest_task("extra2", "win32/opt"),
        unittest_task("crashtest-e10s", "linux/other"),
        unittest_task("gtest", "linux64/asan"),
        talos_task("dromaeojs", "linux64/psan"),
        unittest_task("extra3", "linux/opt"),
        unittest_task("extra4", "linux64/debug", "debug"),
        unittest_task("extra5", "linux/this"),
        unittest_task("extra6", "linux/that"),
        unittest_task("extra7", "linux/other"),
        unittest_task("extra8", "linux64/asan"),
        talos_task("extra9", "linux64/psan"),
    ]
}

RIDEALONG_BUILDS = {
    "linux": ["linux-ridealong"],
    "linux64": ["linux64-ridealong"],
}

GRAPH_CONFIG = {
    "try": {"ridealong-builds": RIDEALONG_BUILDS},
}

for r in RIDEALONG_BUILDS.values():
    tasks.update({k: v for k, v in [unittest_task(n + "-test", n) for n in r]})

unittest_tasks = {
    k: v for k, v in six.iteritems(tasks) if "unittest_try_name" in v.attributes
}
talos_tasks = {
    k: v for k, v in six.iteritems(tasks) if "talos_try_name" in v.attributes
}
graph_with_jobs = TaskGraph(tasks, Graph(set(tasks), set()))


class TestTryOptionSyntax(unittest.TestCase):
    def test_unknown_args(self):
        "unknown arguments are ignored"
        parameters = parse_message("try: --doubledash -z extra")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        # equilvant to "try:"..
        self.assertEqual(tos.build_types, [])
        self.assertEqual(tos.jobs, [])

    def test_apostrophe_in_message(self):
        "apostrophe does not break parsing"
        parameters = parse_message("Increase spammy log's log level. try: -b do")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(sorted(tos.build_types), ["debug", "opt"])

    def test_b_do(self):
        "-b do should produce both build_types"
        parameters = parse_message("try: -b do")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(sorted(tos.build_types), ["debug", "opt"])

    def test_b_d(self):
        "-b d should produce build_types=['debug']"
        parameters = parse_message("try: -b d")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(sorted(tos.build_types), ["debug"])

    def test_b_o(self):
        "-b o should produce build_types=['opt']"
        parameters = parse_message("try: -b o")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(sorted(tos.build_types), ["opt"])

    def test_build_o(self):
        "--build o should produce build_types=['opt']"
        parameters = parse_message("try: --build o")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(sorted(tos.build_types), ["opt"])

    def test_b_dx(self):
        "-b dx should produce build_types=['debug'], silently ignoring the x"
        parameters = parse_message("try: -b dx")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(sorted(tos.build_types), ["debug"])

    def test_j_job(self):
        "-j somejob sets jobs=['somejob']"
        parameters = parse_message("try: -j somejob")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(sorted(tos.jobs), ["somejob"])

    def test_j_jobs(self):
        "-j job1,job2 sets jobs=['job1', 'job2']"
        parameters = parse_message("try: -j job1,job2")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(sorted(tos.jobs), ["job1", "job2"])

    def test_j_all(self):
        "-j all sets jobs=None"
        parameters = parse_message("try: -j all")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(tos.jobs, None)

    def test_j_twice(self):
        "-j job1 -j job2 sets jobs=job1, job2"
        parameters = parse_message("try: -j job1 -j job2")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(sorted(tos.jobs), sorted(["job1", "job2"]))

    def test_p_all(self):
        "-p all sets platforms=None"
        parameters = parse_message("try: -p all")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(tos.platforms, None)

    def test_p_linux(self):
        "-p linux sets platforms=['linux', 'linux-ridealong']"
        parameters = parse_message("try: -p linux")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(tos.platforms, ["linux", "linux-ridealong"])

    def test_p_linux_win32(self):
        "-p linux,win32 sets platforms=['linux', 'linux-ridealong', 'win32']"
        parameters = parse_message("try: -p linux,win32")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(sorted(tos.platforms), ["linux", "linux-ridealong", "win32"])

    def test_p_expands_ridealongs(self):
        "-p linux,linux64 includes the RIDEALONG_BUILDS"
        parameters = parse_message("try: -p linux,linux64")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        platforms = set(["linux"] + RIDEALONG_BUILDS["linux"])
        platforms |= set(["linux64"] + RIDEALONG_BUILDS["linux64"])
        self.assertEqual(sorted(tos.platforms), sorted(platforms))

    def test_u_none(self):
        "-u none sets unittests=[]"
        parameters = parse_message("try: -u none")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(tos.unittests, [])

    def test_u_all(self):
        "-u all sets unittests=[..whole list..]"
        parameters = parse_message("try: -u all")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(tos.unittests, [{"test": t} for t in sorted(unittest_tasks)])

    def test_u_single(self):
        "-u mochitest-webgl1-core sets unittests=[mochitest-webgl1-core]"
        parameters = parse_message("try: -u mochitest-webgl1-core")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(tos.unittests, [{"test": "mochitest-webgl1-core"}])

    def test_u_alias(self):
        "-u mochitest-gl sets unittests=[mochitest-webgl*]"
        parameters = parse_message("try: -u mochitest-gl")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(
            tos.unittests,
            [
                {"test": t}
                for t in [
                    "mochitest-webgl1-core",
                    "mochitest-webgl1-ext",
                    "mochitest-webgl2-core",
                    "mochitest-webgl2-deqp",
                    "mochitest-webgl2-ext",
                ]
            ],
        )

    def test_u_multi_alias(self):
        "-u e10s sets unittests=[all e10s unittests]"
        parameters = parse_message("try: -u e10s")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(
            tos.unittests, [{"test": t} for t in sorted(unittest_tasks) if "e10s" in t]
        )

    def test_u_commas(self):
        "-u mochitest-webgl1-core,gtest sets unittests=both"
        parameters = parse_message("try: -u mochitest-webgl1-core,gtest")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(
            tos.unittests,
            [
                {"test": "gtest"},
                {"test": "mochitest-webgl1-core"},
            ],
        )

    def test_u_chunks(self):
        "-u gtest-3,gtest-4 selects the third and fourth chunk of gtest"
        parameters = parse_message("try: -u gtest-3,gtest-4")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(
            sorted(tos.unittests),
            sorted(
                [
                    {"test": "gtest", "only_chunks": set("34")},
                ]
            ),
        )

    def test_u_platform(self):
        "-u gtest[linux] selects the linux platform for gtest"
        parameters = parse_message("try: -u gtest[linux]")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(
            sorted(tos.unittests),
            sorted(
                [
                    {"test": "gtest", "platforms": ["linux"]},
                ]
            ),
        )

    def test_u_platforms(self):
        "-u gtest[linux,win32] selects the linux and win32 platforms for gtest"
        parameters = parse_message("try: -u gtest[linux,win32]")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(
            sorted(tos.unittests),
            sorted(
                [
                    {"test": "gtest", "platforms": ["linux", "win32"]},
                ]
            ),
        )

    def test_u_platforms_pretty(self):
        """-u gtest[Ubuntu] selects the linux, linux64 and linux64-asan
        platforms for gtest"""
        parameters = parse_message("try: -u gtest[Ubuntu]")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(
            sorted(tos.unittests),
            sorted(
                [
                    {
                        "test": "gtest",
                        "platforms": [
                            "linux32",
                            "linux64",
                            "linux64-asan",
                            "linux1804-64",
                            "linux1804-64-asan",
                        ],
                    },
                ]
            ),
        )

    def test_u_platforms_negated(self):
        "-u gtest[-linux] selects all platforms but linux for gtest"
        parameters = parse_message("try: -u gtest[-linux]")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        all_platforms = set(
            [x.attributes["test_platform"] for x in unittest_tasks.values()]
        )
        self.assertEqual(
            sorted(tos.unittests[0]["platforms"]),
            sorted([x for x in all_platforms if x != "linux"]),
        )

    def test_u_platforms_negated_pretty(self):
        "-u gtest[Ubuntu,-x64] selects just linux for gtest"
        parameters = parse_message("try: -u gtest[Ubuntu,-x64]")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(
            sorted(tos.unittests),
            sorted(
                [
                    {"test": "gtest", "platforms": ["linux32"]},
                ]
            ),
        )

    def test_u_chunks_platforms(self):
        "-u gtest-1[linux,win32] selects the linux and win32 platforms for chunk 1 of gtest"
        parameters = parse_message("try: -u gtest-1[linux,win32]")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(
            tos.unittests,
            [
                {
                    "test": "gtest",
                    "platforms": ["linux", "win32"],
                    "only_chunks": set("1"),
                },
            ],
        )

    def test_t_none(self):
        "-t none sets talos=[]"
        parameters = parse_message("try: -t none")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(tos.talos, [])

    def test_t_all(self):
        "-t all sets talos=[..whole list..]"
        parameters = parse_message("try: -t all")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(tos.talos, [{"test": t} for t in sorted(talos_tasks)])

    def test_t_single(self):
        "-t mochitest-webgl sets talos=[mochitest-webgl]"
        parameters = parse_message("try: -t mochitest-webgl")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(tos.talos, [{"test": "mochitest-webgl"}])

    # -t shares an implementation with -u, so it's not tested heavily

    def test_trigger_tests(self):
        "--rebuild 10 sets trigger_tests"
        parameters = parse_message("try: --rebuild 10")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(tos.trigger_tests, 10)

    def test_talos_trigger_tests(self):
        "--rebuild-talos 10 sets talos_trigger_tests"
        parameters = parse_message("try: --rebuild-talos 10")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(tos.talos_trigger_tests, 10)

    def test_interactive(self):
        "--interactive sets interactive"
        parameters = parse_message("try: --interactive")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(tos.interactive, True)

    def test_all_email(self):
        "--all-emails sets notifications"
        parameters = parse_message("try: --all-emails")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(tos.notifications, "all")

    def test_fail_email(self):
        "--failure-emails sets notifications"
        parameters = parse_message("try: --failure-emails")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(tos.notifications, "failure")

    def test_no_email(self):
        "no email settings don't set notifications"
        parameters = parse_message("try:")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(tos.notifications, None)

    def test_setenv(self):
        "--setenv VAR=value adds a environment variables setting to env"
        parameters = parse_message("try: --setenv VAR1=value1 --setenv VAR2=value2")
        assert parameters["try_task_config"]["env"] == {
            "VAR1": "value1",
            "VAR2": "value2",
        }

    def test_profile(self):
        "--gecko-profile sets profile to true"
        parameters = parse_message("try: --gecko-profile")
        assert parameters["try_task_config"]["gecko-profile"] is True

    def test_tag(self):
        "--tag TAG sets tag to TAG value"
        parameters = parse_message("try: --tag tagName")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertEqual(tos.tag, "tagName")

    def test_no_retry(self):
        "--no-retry sets no_retry to true"
        parameters = parse_message("try: --no-retry")
        tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
        self.assertTrue(tos.no_retry)

    def test_artifact(self):
        "--artifact sets artifact to true"
        parameters = parse_message("try: --artifact")
        assert parameters["try_task_config"]["use-artifact-builds"] is True


if __name__ == "__main__":
    main()