diff options
Diffstat (limited to 'tools/tryselect/test')
| -rw-r--r-- | tools/tryselect/test/test_auto.t | 6 | ||||
| -rw-r--r-- | tools/tryselect/test/test_empty.t | 6 | ||||
| -rw-r--r-- | tools/tryselect/test/test_fuzzy.t | 20 | ||||
| -rw-r--r-- | tools/tryselect/test/test_message.t | 8 | ||||
| -rw-r--r-- | tools/tryselect/test/test_perf.py | 55 | ||||
| -rw-r--r-- | tools/tryselect/test/test_preset.t | 20 | ||||
| -rw-r--r-- | tools/tryselect/test/test_push.py | 30 | ||||
| -rw-r--r-- | tools/tryselect/test/test_release.py | 12 |
8 files changed, 113 insertions, 44 deletions
diff --git a/tools/tryselect/test/test_auto.t b/tools/tryselect/test/test_auto.t index c3fe797949..398eb1a431 100644 --- a/tools/tryselect/test/test_auto.t +++ b/tools/tryselect/test/test_auto.t @@ -8,6 +8,8 @@ Test auto selector Commit message: Tasks automatically selected. + mach try command: `./mach try auto --no-push --no-artifact` + Pushed via `mach try auto` Calculated try_task_config.json: { @@ -27,6 +29,8 @@ Test auto selector Commit message: Tasks automatically selected. ON A CLOSED TREE + mach try command: `./mach try auto --no-push --no-artifact --closed-tree` + Pushed via `mach try auto` Calculated try_task_config.json: { @@ -45,6 +49,8 @@ Test auto selector Commit message: foo Tasks automatically selected. bar ON A CLOSED TREE + mach try command: `./mach try auto --no-push --no-artifact --closed-tree -m "foo {msg} bar"` + Pushed via `mach try auto` Calculated try_task_config.json: { diff --git a/tools/tryselect/test/test_empty.t b/tools/tryselect/test/test_empty.t index d7e9c22618..9e330f8dab 100644 --- a/tools/tryselect/test/test_empty.t +++ b/tools/tryselect/test/test_empty.t @@ -7,6 +7,8 @@ Test empty selector Commit message: No try selector specified, use "Add New Jobs" to select tasks. + mach try command: `./mach try empty --no-push` + Pushed via `mach try empty` Calculated try_task_config.json: { @@ -26,6 +28,8 @@ Test empty selector Commit message: No try selector specified, use "Add New Jobs" to select tasks. ON A CLOSED TREE + mach try command: `./mach try empty --no-push --closed-tree` + Pushed via `mach try empty` Calculated try_task_config.json: { @@ -45,6 +49,8 @@ Test empty selector Commit message: foo No try selector specified, use "Add New Jobs" to select tasks. bar ON A CLOSED TREE + mach try command: `./mach try empty --no-push --closed-tree -m "foo {msg} bar"` + Pushed via `mach try empty` Calculated try_task_config.json: { diff --git a/tools/tryselect/test/test_fuzzy.t b/tools/tryselect/test/test_fuzzy.t index 843b053e08..347e49ce72 100644 --- a/tools/tryselect/test/test_fuzzy.t +++ b/tools/tryselect/test/test_fuzzy.t @@ -7,6 +7,8 @@ Test fuzzy selector Commit message: Fuzzy query='foo + mach try command: `./mach try fuzzy --no-push --no-artifact -q "'foo"` + Pushed via `mach try fuzzy` Calculated try_task_config.json: { @@ -33,6 +35,8 @@ Test fuzzy selector Commit message: Fuzzy query='bar + mach try command: `./mach try fuzzy --no-push --no-artifact --full -q "'bar"` + Pushed via `mach try fuzzy` Calculated try_task_config.json: { @@ -58,6 +62,8 @@ Test multiple selectors Commit message: Fuzzy query='foo&query='bar + mach try command: `./mach try fuzzy --no-push --no-artifact --full -q "'foo" -q "'bar"` + Pushed via `mach try fuzzy` Calculated try_task_config.json: { @@ -85,6 +91,8 @@ Test query intersection Commit message: Fuzzy query='foo&query='opt + mach try command: `./mach try fuzzy --no-push --no-artifact --and -q "'foo" -q "'opt"` + Pushed via `mach try fuzzy` Calculated try_task_config.json: { @@ -112,6 +120,8 @@ Test intersection with preset containing multiple queries Commit message: Fuzzy query='test&query='opt&query='test + mach try command: `./mach try fuzzy --no-push --no-artifact --preset foo -xq "'test"` + Pushed via `mach try fuzzy` Calculated try_task_config.json: { @@ -134,6 +144,8 @@ Test intersection with preset containing multiple queries Commit message: Fuzzy query='test&query='opt&query='test + mach try command: `./mach try --no-push --no-artifact --preset foo -xq "'test"` + Pushed via `mach try fuzzy` Calculated try_task_config.json: { @@ -159,6 +171,8 @@ Test exact match Commit message: Fuzzy query=testfoo | 'testbar + mach try command: `./mach try fuzzy --no-push --no-artifact --full -q "testfoo | 'testbar"` + Pushed via `mach try fuzzy` Calculated try_task_config.json: { @@ -181,6 +195,8 @@ Test exact match Commit message: Fuzzy query=testfoo | 'testbar + mach try command: `./mach try fuzzy --no-push --no-artifact --full --exact -q "testfoo | 'testbar"` + Pushed via `mach try fuzzy` Calculated try_task_config.json: { @@ -206,6 +222,8 @@ Test task config Commit message: Fuzzy query='foo + mach try command: `./mach try fuzzy --no-push --artifact -q "'foo"` + Pushed via `mach try fuzzy` Calculated try_task_config.json: { @@ -230,6 +248,8 @@ Test task config Commit message: Fuzzy query='foo + mach try command: `./mach try fuzzy --no-push --no-artifact --env FOO=1 --env BAR=baz -q "'foo"` + Pushed via `mach try fuzzy` Calculated try_task_config.json: { diff --git a/tools/tryselect/test/test_message.t b/tools/tryselect/test/test_message.t index a707e410fb..45adc2dc5a 100644 --- a/tools/tryselect/test/test_message.t +++ b/tools/tryselect/test/test_message.t @@ -9,6 +9,8 @@ Test custom commit messages with fuzzy selector Fuzzy query=foo + mach try command: `./mach try fuzzy --no-push --no-artifact -q foo --message Foobar` + Pushed via `mach try fuzzy` Calculated try_task_config.json: { @@ -31,6 +33,8 @@ Test custom commit messages with fuzzy selector Commit message: Foobar: Fuzzy query=foo + mach try command: `./mach try fuzzy --no-push --no-artifact -q foo -m "Foobar: {msg}"` + Pushed via `mach try fuzzy` Calculated try_task_config.json: { @@ -62,11 +66,15 @@ Test custom commit messages with syntax selector try: -b do -p linux -u mochitests + mach try command: `./mach try syntax --no-push --no-artifact -p linux -u mochitests --message Foobar` + Pushed via `mach try syntax` $ ./mach try syntax $testargs -p linux -u mochitests -m "Foobar: {msg}" Commit message: Foobar: try: -b do -p linux -u mochitests + mach try command: `./mach try syntax --no-push --no-artifact -p linux -u mochitests -m "Foobar: {msg}"` + Pushed via `mach try syntax` $ unset EDITOR $ ./mach try syntax $testargs -p linux -u mochitests -m > /dev/null 2>&1 diff --git a/tools/tryselect/test/test_perf.py b/tools/tryselect/test/test_perf.py index c4e3b6b5fa..73aa3c4d71 100644 --- a/tools/tryselect/test/test_perf.py +++ b/tools/tryselect/test/test_perf.py @@ -193,7 +193,6 @@ TEST_CATEGORIES = { "!live", "!profil", "!chrom", - "!fenix", "!safari", "!m-car", ] @@ -206,7 +205,6 @@ TEST_CATEGORIES = { "!live", "!profil", "!chrom", - "!fenix", "!safari", "!m-car", ] @@ -220,7 +218,6 @@ TEST_CATEGORIES = { "!live", "!profil", "!chrom", - "!fenix", "!safari", "!m-car", ], @@ -234,7 +231,6 @@ TEST_CATEGORIES = { }, [ "Responsiveness android-p2 geckoview", - "Benchmarks desktop chromium", ], ), # Default settings ( @@ -248,7 +244,6 @@ TEST_CATEGORIES = { "!bytecode", "!profil", "!chrom", - "!fenix", "!safari", "!m-car", ] @@ -260,7 +255,6 @@ TEST_CATEGORIES = { "!bytecode", "!profil", "!chrom", - "!fenix", "!safari", "!m-car", ] @@ -273,7 +267,6 @@ TEST_CATEGORIES = { "!bytecode", "!profil", "!chrom", - "!fenix", "!safari", "!m-car", ], @@ -281,7 +274,6 @@ TEST_CATEGORIES = { }, [ "Responsiveness android-p2 geckoview", - "Benchmarks desktop chromium", "Benchmarks desktop firefox profiling", "Talos desktop live-sites", "Talos desktop profiling+swr", @@ -300,7 +292,6 @@ TEST_CATEGORIES = { "!bytecode", "!profil", "!chrom", - "!fenix", "!m-car", ] }, @@ -331,7 +322,7 @@ TEST_CATEGORIES = { ), ( {"live_sites": True, "chrome": True}, - 114, + 90, { "Benchmarks desktop": { "raptor": [ @@ -339,7 +330,6 @@ TEST_CATEGORIES = { "!android 'shippable !-32 !clang", "!bytecode", "!profil", - "!fenix", "!safari", "!m-car", ] @@ -351,20 +341,10 @@ TEST_CATEGORIES = { "'live", "!bytecode", "!profil", - "!fenix", "!safari", "!m-car", ], }, - "Benchmarks desktop chromium": { - "raptor": [ - "'browsertime 'benchmark", - "!android 'shippable !-32 !clang", - "'chromium", - "!bytecode", - "!profil", - ], - }, }, [ "Responsiveness android-p2 geckoview", @@ -374,7 +354,7 @@ TEST_CATEGORIES = { ), ( {"android": True}, - 78, + 88, { "Benchmarks desktop": { "raptor": [ @@ -384,7 +364,6 @@ TEST_CATEGORIES = { "!live", "!profil", "!chrom", - "!fenix", "!safari", "!m-car", ], @@ -403,12 +382,11 @@ TEST_CATEGORIES = { [ "Responsiveness android-a51 chrome-m", "Firefox Pageload android", - "Pageload android-a51 fenix", ], ), ( {"android": True, "chrome": True}, - 128, + 118, { "Benchmarks desktop": { "raptor": [ @@ -417,7 +395,6 @@ TEST_CATEGORIES = { "!bytecode", "!live", "!profil", - "!fenix", "!safari", "!m-car", ], @@ -437,7 +414,7 @@ TEST_CATEGORIES = { ), ( {"android": True, "chrome": True, "profile": True}, - 164, + 156, { "Benchmarks desktop": { "raptor": [ @@ -445,7 +422,6 @@ TEST_CATEGORIES = { "!android 'shippable !-32 !clang", "!bytecode", "!live", - "!fenix", "!safari", "!m-car", ] @@ -463,7 +439,6 @@ TEST_CATEGORIES = { "Resource Usage desktop profiling", "DAMP (Devtools) desktop chrome", "Resource Usage android", - "Resource Usage windows chromium", ], ), ( @@ -497,7 +472,6 @@ TEST_CATEGORIES = { "Resource Usage desktop profiling", "DAMP (Devtools) desktop chrome", "Resource Usage android", - "Resource Usage windows chromium", ], ), # Show all available windows tests, no other platform should exist @@ -755,7 +729,6 @@ TEST_CATEGORIES = { "!bytecode", "!profil", "!chrom", - "!fenix", "!safari", "!m-car", ], @@ -767,7 +740,6 @@ TEST_CATEGORIES = { "!bytecode", "!profil", "!chrom", - "!fenix", "!safari", "!m-car", ], @@ -847,7 +819,7 @@ def test_category_expansion_with_non_pgo_flag(category_options, call_counts): [ ( {}, - [10, 2, 2, 10, 2, 1], + [9, 2, 2, 10, 2, 1], 2, ( "\n!!!NOTE!!!\n You'll be able to find a performance comparison " @@ -858,7 +830,7 @@ def test_category_expansion_with_non_pgo_flag(category_options, call_counts): ), ( {"query": "'Pageload 'linux 'firefox"}, - [10, 2, 2, 10, 2, 1], + [9, 2, 2, 10, 2, 1], 2, ( "\n!!!NOTE!!!\n You'll be able to find a performance comparison " @@ -869,7 +841,7 @@ def test_category_expansion_with_non_pgo_flag(category_options, call_counts): ), ( {"cached_revision": "cached_base_revision"}, - [10, 1, 1, 10, 2, 0], + [9, 1, 1, 10, 2, 0], 2, ( "\n!!!NOTE!!!\n You'll be able to find a performance comparison " @@ -880,7 +852,7 @@ def test_category_expansion_with_non_pgo_flag(category_options, call_counts): ), ( {"dry_run": True}, - [10, 1, 1, 10, 2, 0], + [9, 1, 1, 10, 2, 0], 2, ( "\n!!!NOTE!!!\n You'll be able to find a performance comparison " @@ -913,7 +885,7 @@ def test_category_expansion_with_non_pgo_flag(category_options, call_counts): ), ( {"single_run": True}, - [10, 1, 1, 4, 2, 0], + [9, 1, 1, 4, 2, 0], 2, ( "If you need any help, you can find us in the #perf-help Matrix channel:\n" @@ -922,7 +894,7 @@ def test_category_expansion_with_non_pgo_flag(category_options, call_counts): ), ( {"detect_changes": True}, - [11, 2, 2, 10, 2, 1], + [10, 2, 2, 10, 2, 1], 2, ( "\n!!!NOTE!!!\n You'll be able to find a performance comparison " @@ -933,7 +905,7 @@ def test_category_expansion_with_non_pgo_flag(category_options, call_counts): ), ( {"perfcompare_beta": True}, - [10, 2, 2, 10, 2, 1], + [9, 2, 2, 10, 2, 1], 2, ( "\n!!!NOTE!!!\n You'll be able to find a performance comparison " @@ -971,7 +943,6 @@ def test_full_run(options, call_counts, log_ind, expected_log_message): ["", TASKS], ["", TASKS], ["", TASKS], - ["", TASKS], ["", ["Perftest Change Detector"]], ] # Number of side effects for fzf should always be greater than @@ -997,11 +968,11 @@ def test_full_run(options, call_counts, log_ind, expected_log_message): [ ( {"detect_changes": True}, - [11, 0, 0, 2, 1], + [10, 0, 0, 2, 1], 1, ( "Executing raptor queries: 'browsertime 'benchmark, !clang 'linux " - "'shippable, !bytecode, !live, !profil, !chrom, !fenix, !safari, !m-car" + "'shippable, !bytecode, !live, !profil, !chrom, !safari, !m-car" ), InvalidRegressionDetectorQuery, ), diff --git a/tools/tryselect/test/test_preset.t b/tools/tryselect/test/test_preset.t index 13e6946d32..07b4fb1bf4 100644 --- a/tools/tryselect/test/test_preset.t +++ b/tools/tryselect/test/test_preset.t @@ -10,12 +10,16 @@ Test preset with no subcommand Commit message: try: -b do -p linux -u mochitests -t none --tag foo + mach try command: `./mach try --no-push --no-artifact --preset foo` + Pushed via `mach try syntax` $ ./mach try syntax $testargs --preset foo Commit message: try: -b do -p linux -u mochitests -t none --tag foo + mach try command: `./mach try syntax --no-push --no-artifact --preset foo` + Pushed via `mach try syntax` $ ./mach try $testargs --list-presets @@ -59,12 +63,16 @@ Test preset with syntax subcommand Commit message: try: -b do -p win32 -u none -t all --tag bar + mach try command: `./mach try syntax --no-push --no-artifact --preset bar` + Pushed via `mach try syntax` $ ./mach try $testargs --preset bar Commit message: try: -b do -p win32 -u none -t all --tag bar + mach try command: `./mach try --no-push --no-artifact --preset bar` + Pushed via `mach try syntax` $ ./mach try syntax $testargs --list-presets @@ -128,6 +136,8 @@ Test preset with fuzzy subcommand Commit message: Fuzzy query='foo + mach try command: `./mach try fuzzy --no-push --no-artifact --preset baz` + Pushed via `mach try fuzzy` Calculated try_task_config.json: { @@ -152,6 +162,8 @@ Test preset with fuzzy subcommand Commit message: Fuzzy query='foo + mach try command: `./mach try --no-push --no-artifact --preset baz` + Pushed via `mach try fuzzy` Calculated try_task_config.json: { @@ -178,6 +190,8 @@ Queries can be appended to presets Commit message: Fuzzy query='foo&query='build + mach try command: `./mach try fuzzy --no-push --no-artifact --preset baz -q "'build"` + Pushed via `mach try fuzzy` Calculated try_task_config.json: { @@ -203,6 +217,8 @@ Queries can be appended to presets Commit message: Fuzzy query='foo&query='opt + mach try command: `./mach try --no-push --no-artifact --preset baz -xq "'opt"` + Pushed via `mach try fuzzy` Calculated try_task_config.json: { @@ -294,6 +310,8 @@ Test gecko-profile argument handling. Add in profiling to a preset. Commit message: Fuzzy query='foo + mach try command: `./mach try fuzzy --no-push --no-artifact --preset baz --gecko-profile-features=nostacksampling,cpu` + Pushed via `mach try fuzzy` Calculated try_task_config.json: { @@ -326,6 +344,8 @@ settings; everything else uses dashes.) Commit message: Fuzzy query='foo + mach try command: `./mach try fuzzy --no-push --no-artifact --preset profile` + Pushed via `mach try fuzzy` Calculated try_task_config.json: { diff --git a/tools/tryselect/test/test_push.py b/tools/tryselect/test/test_push.py index 97f2e047d7..c8dbbe4184 100644 --- a/tools/tryselect/test/test_push.py +++ b/tools/tryselect/test/test_push.py @@ -50,5 +50,35 @@ def test_generate_try_task_config(method, labels, params, routes, expected): ) +def test_get_sys_argv(): + input_argv = [ + "./mach", + "try", + "fuzzy", + "--full", + "--artifact", + "--push-to-lando", + "--query", + "'android-hw !shippable !nofis", + "--no-push", + ] + expected_string = './mach try fuzzy --full --artifact --push-to-lando --query "\'android-hw !shippable !nofis" --no-push' + assert push.get_sys_argv(input_argv) == expected_string + + +def test_get_sys_argv_2(): + input_argv = [ + "./mach", + "try", + "fuzzy", + "--query", + "'test-linux1804-64-qr/opt-mochitest-plain-", + "--worker-override=t-linux-large=gecko-t/t-linux-2204-wayland-experimental", + "--no-push", + ] + expected_string = './mach try fuzzy --query "\'test-linux1804-64-qr/opt-mochitest-plain-" --worker-override=t-linux-large=gecko-t/t-linux-2204-wayland-experimental --no-push' + assert push.get_sys_argv(input_argv) == expected_string + + if __name__ == "__main__": mozunit.main() diff --git a/tools/tryselect/test/test_release.py b/tools/tryselect/test/test_release.py index a1a0d348b2..00eeb0c6c6 100644 --- a/tools/tryselect/test/test_release.py +++ b/tools/tryselect/test/test_release.py @@ -18,11 +18,18 @@ def test_release(run_mach, capfd): output = capfd.readouterr().out print(output) - expected = dedent( + expected_part1 = dedent( """ Commit message: staging release: 97.0 + """ + ).lstrip() + # The output now features a display of the `mach try` command run here + # that will vary based on the user and invocation, so ignore that part. + + expected_part2 = dedent( + """ Pushed via `mach try release` Calculated try_task_config.json: { @@ -36,7 +43,8 @@ def test_release(run_mach, capfd): """ ).lstrip() - assert expected in output + assert expected_part1 in output + assert expected_part2 in output if __name__ == "__main__": |