Adding upstream version 110.0.1.upstream/110.0.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 854 insertions, 0 deletions

diff --git a/tools/tryselect/test/test_perf.py b/tools/tryselect/test/test_perf.py
new file mode 100644
index 0000000000..b3ad5d9575
--- /dev/null
+++ b/tools/tryselect/test/test_perf.py
@@ -0,0 +1,854 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+from unittest import mock
+
+import mozunit
+import pytest
+from tryselect.selectors.perf import (
+    Apps,
+    InvalidCategoryException,
+    PerfParser,
+    Platforms,
+    Suites,
+    Variants,
+    check_for_live_sites,
+    check_for_profile,
+    run,
+)
+
+TASKS = [
+    "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-motionmark-animometer",
+    "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot-optimizing",
+    "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-webaudio",
+    "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-speedometer",
+    "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-misc",
+    "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-jetstream2",
+    "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-ares6",
+    "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-misc-optimizing",
+    "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-sunspider",
+    "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-matrix-react-bench",
+    "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot-baseline",
+    "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-twitch-animation",
+    "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-assorted-dom",
+    "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-stylebench",
+    "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-misc-baseline",
+    "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-motionmark-htmlsuite",
+    "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-unity-webgl",
+    "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot",
+]
+
+# The TEST_VARIANTS, and TEST_CATEGORIES are used to force
+# a particular set of categories to show up in testing. Otherwise,
+# every time someone adds a category, or a variant, we'll need
+# to redo all the category counts. The platforms, and apps are
+# not forced because they change infrequently.
+TEST_VARIANTS = {
+    Variants.NO_FISSION.value: {
+        "query": "'nofis",
+        "negation": "!nofis",
+        "platforms": [Platforms.ANDROID.value],
+        "apps": [Apps.FENIX.value, Apps.GECKOVIEW.value],
+    },
+    Variants.BYTECODE_CACHED.value: {
+        "query": "'bytecode",
+        "negation": "!bytecode",
+        "platforms": [Platforms.DESKTOP.value],
+        "apps": [Apps.FIREFOX.value],
+    },
+    Variants.LIVE_SITES.value: {
+        "query": "'live",
+        "negation": "!live",
+        "restriction": check_for_live_sites,
+        "platforms": [Platforms.DESKTOP.value, Platforms.ANDROID.value],
+        "apps": list(PerfParser.apps.keys()),
+    },
+    Variants.PROFILING.value: {
+        "query": "'profil",
+        "negation": "!profil",
+        "restriction": check_for_profile,
+        "platforms": [Platforms.DESKTOP.value, Platforms.ANDROID.value],
+        "apps": [Apps.FIREFOX.value, Apps.GECKOVIEW.value, Apps.FENIX.value],
+    },
+    Variants.SWR.value: {
+        "query": "'swr",
+        "negation": "!swr",
+        "platforms": [Platforms.DESKTOP.value],
+        "apps": [Apps.FIREFOX.value],
+    },
+}
+
+TEST_CATEGORIES = {
+    "Pageload": {
+        "query": {
+            Suites.RAPTOR.value: ["'browsertime 'tp6"],
+        },
+        "suites": [Suites.RAPTOR.value],
+        "tasks": [],
+    },
+    "Pageload (essential)": {
+        "query": {
+            Suites.RAPTOR.value: ["'browsertime 'tp6 'essential"],
+        },
+        "variant-restrictions": {Suites.RAPTOR.value: [Variants.NO_FISSION.value]},
+        "suites": [Suites.RAPTOR.value],
+        "tasks": [],
+    },
+    "Responsiveness": {
+        "query": {
+            Suites.RAPTOR.value: ["'browsertime 'responsive"],
+        },
+        "suites": [Suites.RAPTOR.value],
+        "variant-restrictions": {Suites.RAPTOR.value: []},
+        "tasks": [],
+    },
+    "Benchmarks": {
+        "query": {
+            Suites.RAPTOR.value: ["'browsertime 'benchmark"],
+        },
+        "suites": [Suites.RAPTOR.value],
+        "variant-restrictions": {Suites.RAPTOR.value: []},
+        "tasks": [],
+    },
+    "DAMP (Devtools)": {
+        "query": {
+            Suites.TALOS.value: ["'talos 'damp"],
+        },
+        "suites": [Suites.TALOS.value],
+        "tasks": [],
+    },
+    "Talos PerfTests": {
+        "query": {
+            Suites.TALOS.value: ["'talos"],
+        },
+        "suites": [Suites.TALOS.value],
+        "tasks": [],
+    },
+    "Resource Usage": {
+        "query": {
+            Suites.TALOS.value: ["'talos 'xperf | 'tp5"],
+            Suites.RAPTOR.value: ["'power 'osx"],
+            Suites.AWSY.value: ["'awsy"],
+        },
+        "suites": [Suites.TALOS.value, Suites.RAPTOR.value, Suites.AWSY.value],
+        "platform-restrictions": [Platforms.DESKTOP.value],
+        "variant-restrictions": {
+            Suites.RAPTOR.value: [],
+            Suites.TALOS.value: [],
+        },
+        "app-restrictions": {
+            Suites.RAPTOR.value: [Apps.FIREFOX.value],
+            Suites.TALOS.value: [Apps.FIREFOX.value],
+        },
+        "tasks": [],
+    },
+    "Graphics, & Media Playback": {
+        "query": {
+            # XXX This might not be an exhaustive list for talos atm
+            Suites.TALOS.value: ["'talos 'svgr | 'bcv | 'webgl"],
+            Suites.RAPTOR.value: ["'browsertime 'youtube-playback"],
+        },
+        "suites": [Suites.TALOS.value, Suites.RAPTOR.value],
+        "variant-restrictions": {Suites.RAPTOR.value: [Variants.NO_FISSION.value]},
+        "tasks": [],
+    },
+}
+
+
+@pytest.mark.parametrize(
+    "category_options, expected_counts, unique_categories, missing",
+    [
+        # Default should show the premade live category, but no chrome or android
+        # The benchmark desktop category should be visible in all configurations
+        # except for when there are requested apps/variants/platforms
+        (
+            {},
+            58,
+            {
+                "Benchmarks desktop": {
+                    "raptor": [
+                        "'browsertime 'benchmark",
+                        "!android 'shippable !-32 !clang",
+                        "!bytecode",
+                        "!live",
+                        "!profil",
+                        "!chrom",
+                        "!safari",
+                    ]
+                },
+                "Pageload macosx": {
+                    "raptor": [
+                        "'browsertime 'tp6",
+                        "'osx 'shippable",
+                        "!bytecode",
+                        "!live",
+                        "!profil",
+                        "!chrom",
+                        "!safari",
+                    ]
+                },
+                "Resource Usage desktop": {
+                    "awsy": ["'awsy", "!android 'shippable !-32 !clang"],
+                    "raptor": [
+                        "'power 'osx",
+                        "!android 'shippable !-32 !clang",
+                        "!bytecode",
+                        "!live",
+                        "!profil",
+                        "!chrom",
+                        "!safari",
+                    ],
+                    "talos": [
+                        "'talos 'xperf | 'tp5",
+                        "!android 'shippable !-32 !clang",
+                        "!profil",
+                        "!swr",
+                    ],
+                },
+            },
+            [
+                "Responsiveness android-p2 geckoview",
+                "Benchmarks desktop chromium",
+            ],
+        ),  # Default settings
+        (
+            {"live_sites": True},
+            66,
+            {
+                "Benchmarks desktop": {
+                    "raptor": [
+                        "'browsertime 'benchmark",
+                        "!android 'shippable !-32 !clang",
+                        "!bytecode",
+                        "!profil",
+                        "!chrom",
+                        "!safari",
+                    ]
+                },
+                "Pageload macosx": {
+                    "raptor": [
+                        "'browsertime 'tp6",
+                        "'osx 'shippable",
+                        "!bytecode",
+                        "!profil",
+                        "!chrom",
+                        "!safari",
+                    ]
+                },
+                "Pageload macosx live-sites": {
+                    "raptor": [
+                        "'browsertime 'tp6",
+                        "'osx 'shippable",
+                        "'live",
+                        "!bytecode",
+                        "!profil",
+                        "!chrom",
+                        "!safari",
+                    ],
+                },
+            },
+            [
+                "Responsiveness android-p2 geckoview",
+                "Benchmarks desktop chromium",
+                "Benchmarks desktop firefox profiling",
+                "Talos desktop live-sites",
+                "Talos desktop profiling+swr",
+                "Benchmarks desktop firefox live-sites+profiling"
+                "Benchmarks desktop firefox live-sites",
+            ],
+        ),
+        (
+            {"live_sites": True, "safari": True},
+            72,
+            {
+                "Benchmarks desktop": {
+                    "raptor": [
+                        "'browsertime 'benchmark",
+                        "!android 'shippable !-32 !clang",
+                        "!bytecode",
+                        "!profil",
+                        "!chrom",
+                    ]
+                },
+                "Pageload macosx safari": {
+                    "raptor": [
+                        "'browsertime 'tp6",
+                        "'osx 'shippable",
+                        "'safari",
+                        "!bytecode",
+                        "!profil",
+                    ]
+                },
+                "Pageload macosx safari live-sites": {
+                    "raptor": [
+                        "'browsertime 'tp6",
+                        "'osx 'shippable",
+                        "'safari",
+                        "'live",
+                        "!bytecode",
+                        "!profil",
+                    ],
+                },
+            },
+            [
+                "Pageload linux safari",
+                "Pageload desktop safari",
+            ],
+        ),
+        (
+            {"live_sites": True, "chrome": True},
+            114,
+            {
+                "Benchmarks desktop": {
+                    "raptor": [
+                        "'browsertime 'benchmark",
+                        "!android 'shippable !-32 !clang",
+                        "!bytecode",
+                        "!profil",
+                        "!safari",
+                    ]
+                },
+                "Pageload macosx live-sites": {
+                    "raptor": [
+                        "'browsertime 'tp6",
+                        "'osx 'shippable",
+                        "'live",
+                        "!bytecode",
+                        "!profil",
+                        "!safari",
+                    ],
+                },
+                "Benchmarks desktop chromium": {
+                    "raptor": [
+                        "'browsertime 'benchmark",
+                        "!android 'shippable !-32 !clang",
+                        "'chromium",
+                        "!bytecode",
+                        "!profil",
+                    ],
+                },
+            },
+            [
+                "Responsiveness android-p2 geckoview",
+                "Firefox Pageload linux chrome",
+                "Talos PerfTests desktop swr",
+            ],
+        ),
+        (
+            {"android": True},
+            88,
+            {
+                "Benchmarks desktop": {
+                    "raptor": [
+                        "'browsertime 'benchmark",
+                        "!android 'shippable !-32 !clang",
+                        "!bytecode",
+                        "!live",
+                        "!profil",
+                        "!chrom",
+                        "!safari",
+                    ],
+                },
+                "Responsiveness android-a51 geckoview": {
+                    "raptor": [
+                        "'browsertime 'responsive",
+                        "'android 'a51 'shippable 'aarch64",
+                        "'geckoview",
+                        "!nofis",
+                        "!live",
+                        "!profil",
+                    ],
+                },
+            },
+            [
+                "Responsiveness android-a51 chrome-m",
+                "Firefox Pageload android",
+            ],
+        ),
+        (
+            {"android": True, "chrome": True},
+            138,
+            {
+                "Benchmarks desktop": {
+                    "raptor": [
+                        "'browsertime 'benchmark",
+                        "!android 'shippable !-32 !clang",
+                        "!bytecode",
+                        "!live",
+                        "!profil",
+                        "!safari",
+                    ],
+                },
+                "Responsiveness android-a51 chrome-m": {
+                    "raptor": [
+                        "'browsertime 'responsive",
+                        "'android 'a51 'shippable 'aarch64",
+                        "'chrome-m",
+                        "!nofis",
+                        "!live",
+                        "!profil",
+                    ],
+                },
+            },
+            ["Responsiveness android-p2 chrome-m", "Resource Usage android"],
+        ),
+        (
+            {"android": True, "chrome": True, "profile": True},
+            176,
+            {
+                "Benchmarks desktop": {
+                    "raptor": [
+                        "'browsertime 'benchmark",
+                        "!android 'shippable !-32 !clang",
+                        "!bytecode",
+                        "!live",
+                        "!safari",
+                    ]
+                },
+                "Talos PerfTests desktop profiling": {
+                    "talos": [
+                        "'talos",
+                        "!android 'shippable !-32 !clang",
+                        "'profil",
+                        "!swr",
+                    ]
+                },
+            },
+            [
+                "Resource Usage desktop profiling",
+                "DAMP (Devtools) desktop chrome",
+                "Resource Usage android",
+                "Resource Usage windows chromium",
+            ],
+        ),
+        # Show all available windows tests, no other platform should exist
+        # including the desktop catgeory
+        (
+            {"requested_platforms": ["windows"]},
+            14,
+            {
+                "Benchmarks windows firefox": {
+                    "raptor": [
+                        "'browsertime 'benchmark",
+                        "!-32 'windows 'shippable",
+                        "!chrom !geckoview !fenix !safari",
+                        "!bytecode",
+                        "!live",
+                        "!profil",
+                    ]
+                },
+            },
+            [
+                "Resource Usage desktop",
+                "Benchmarks desktop",
+                "Benchmarks linux firefox bytecode-cached+profiling",
+            ],
+        ),
+        # Can't have fenix on the windows platform
+        (
+            {"requested_platforms": ["windows"], "requested_apps": ["fenix"]},
+            0,
+            {},
+            ["Benchmarks desktop"],
+        ),
+        # Android flag also needs to be supplied
+        (
+            {"requested_platforms": ["android"], "requested_apps": ["fenix"]},
+            0,
+            {},
+            ["Benchmarks desktop"],
+        ),
+        # There should be no global categories available, only fenix
+        (
+            {
+                "requested_platforms": ["android"],
+                "requested_apps": ["fenix"],
+                "android": True,
+            },
+            10,
+            {
+                "Pageload android fenix": {
+                    "raptor": [
+                        "'browsertime 'tp6",
+                        "'android 'a51 'shippable 'aarch64",
+                        "'fenix",
+                        "!nofis",
+                        "!live",
+                        "!profil",
+                    ],
+                }
+            },
+            ["Benchmarks desktop", "Pageload (live) android"],
+        ),
+        # Test with multiple apps
+        (
+            {
+                "requested_platforms": ["android"],
+                "requested_apps": ["fenix", "geckoview"],
+                "android": True,
+            },
+            15,
+            {
+                "Benchmarks android geckoview": {
+                    "raptor": [
+                        "'browsertime 'benchmark",
+                        "'android 'a51 'shippable 'aarch64",
+                        "'geckoview",
+                        "!nofis",
+                        "!live",
+                        "!profil",
+                    ],
+                },
+                "Pageload android fenix": {
+                    "raptor": [
+                        "'browsertime 'tp6",
+                        "'android 'a51 'shippable 'aarch64",
+                        "'fenix",
+                        "!nofis",
+                        "!live",
+                        "!profil",
+                    ],
+                },
+            },
+            [
+                "Benchmarks desktop",
+                "Pageload android no-fission",
+                "Pageload android fenix live-sites",
+            ],
+        ),
+        # Variants are inclusive, so we'll see the variant alongside the
+        # base here for fenix
+        (
+            {
+                "requested_variants": ["no-fission"],
+                "requested_apps": ["fenix"],
+                "android": True,
+            },
+            32,
+            {
+                "Pageload android-a51 fenix": {
+                    "raptor": [
+                        "'browsertime 'tp6",
+                        "'android 'a51 'shippable 'aarch64",
+                        "'fenix",
+                        "!live",
+                        "!profil",
+                    ],
+                },
+                "Pageload android-a51 fenix no-fission": {
+                    "raptor": [
+                        "'browsertime 'tp6",
+                        "'android 'a51 'shippable 'aarch64",
+                        "'fenix",
+                        "'nofis",
+                        "!live",
+                        "!profil",
+                    ],
+                },
+                "Pageload (essential) android fenix no-fission": {
+                    "raptor": [
+                        "'browsertime 'tp6 'essential",
+                        "'android 'a51 'shippable 'aarch64",
+                        "'fenix",
+                        "'nofis",
+                        "!live",
+                        "!profil",
+                    ],
+                },
+            },
+            [
+                "Benchmarks desktop",
+                "Pageload (live) android",
+                "Pageload android-p2 fenix live-sites",
+            ],
+        ),
+        # With multiple variants, we'll see the base variant (with no combinations)
+        # for each of them
+        (
+            {
+                "requested_variants": ["no-fission", "live-sites"],
+                "requested_apps": ["fenix"],
+                "android": True,
+            },
+            40,
+            {
+                "Pageload android-a51 fenix": {
+                    "raptor": [
+                        "'browsertime 'tp6",
+                        "'android 'a51 'shippable 'aarch64",
+                        "'fenix",
+                        "!profil",
+                    ],
+                },
+                "Pageload android-a51 fenix no-fission": {
+                    "raptor": [
+                        "'browsertime 'tp6",
+                        "'android 'a51 'shippable 'aarch64",
+                        "'fenix",
+                        "'nofis",
+                        "!live",
+                        "!profil",
+                    ],
+                },
+                "Pageload android-a51 fenix live-sites": {
+                    "raptor": [
+                        "'browsertime 'tp6",
+                        "'android 'a51 'shippable 'aarch64",
+                        "'fenix",
+                        "'live",
+                        "!nofis",
+                        "!profil",
+                    ],
+                },
+                "Pageload (essential) android fenix no-fission": {
+                    "raptor": [
+                        "'browsertime 'tp6 'essential",
+                        "'android 'a51 'shippable 'aarch64",
+                        "'fenix",
+                        "'nofis",
+                        "!live",
+                        "!profil",
+                    ],
+                },
+                "Pageload android fenix no-fission+live-sites": {
+                    "raptor": [
+                        "'browsertime 'tp6",
+                        "'android 'a51 'shippable 'aarch64",
+                        "'fenix",
+                        "'nofis",
+                        "'live",
+                        "!profil",
+                    ],
+                },
+            },
+            [
+                "Benchmarks desktop",
+                "Pageload (live) android",
+                "Pageload android-p2 fenix live-sites",
+                "Pageload (essential) android fenix no-fission+live-sites",
+            ],
+        ),
+        # Make sure that no no-fission tasks are selected when a variant cannot
+        # run on a requested platform
+        (
+            {
+                "requested_variants": ["no-fission"],
+                "requested_platforms": ["windows"],
+            },
+            14,
+            {
+                "Responsiveness windows firefox": {
+                    "raptor": [
+                        "'browsertime 'responsive",
+                        "!-32 'windows 'shippable",
+                        "!chrom !geckoview !fenix !safari",
+                        "!bytecode",
+                        "!live",
+                        "!profil",
+                    ],
+                },
+            },
+            ["Benchmarks desktop", "Responsiveness windows firefox no-fisson"],
+        ),
+        # We should only see the base and the live-site variants here for windows
+        (
+            {
+                "requested_variants": ["no-fission", "live-sites"],
+                "requested_platforms": ["windows"],
+                "android": True,
+            },
+            16,
+            {
+                "Responsiveness windows firefox": {
+                    "raptor": [
+                        "'browsertime 'responsive",
+                        "!-32 'windows 'shippable",
+                        "!chrom !geckoview !fenix !safari",
+                        "!bytecode",
+                        "!profil",
+                    ],
+                },
+                "Pageload windows live-sites": {
+                    "raptor": [
+                        "'browsertime 'tp6",
+                        "!-32 'windows 'shippable",
+                        "'live",
+                        "!bytecode",
+                        "!profil",
+                        "!chrom",
+                        "!safari",
+                    ],
+                },
+                "Graphics, & Media Playback windows": {
+                    "raptor": [
+                        "'browsertime 'youtube-playback",
+                        "!-32 'windows 'shippable",
+                        "!bytecode",
+                        "!profil",
+                        "!chrom",
+                        "!safari",
+                    ],
+                    "talos": [
+                        "'talos 'svgr | 'bcv | 'webgl",
+                        "!-32 'windows 'shippable",
+                        "!profil",
+                        "!swr",
+                    ],
+                },
+            },
+            [
+                "Benchmarks desktop",
+                "Responsiveness windows firefox no-fisson",
+                "Pageload (live) android",
+                "Talos desktop live-sites",
+                "Talos android",
+                "Graphics, & Media Playback windows live-sites",
+                "Graphics, & Media Playback android no-fission",
+            ],
+        ),
+    ],
+)
+def test_category_expansion(
+    category_options, expected_counts, unique_categories, missing
+):
+    # Set the categories, and variants to expand
+    PerfParser.categories = TEST_CATEGORIES
+    PerfParser.variants = TEST_VARIANTS
+
+    # Expand the categories, then either check if the unique_categories,
+    # exist or are missing from the categories
+    expanded_cats = PerfParser.get_categories(**category_options)
+
+    assert len(expanded_cats) == expected_counts
+    assert not any([expanded_cats.get(ucat, None) is not None for ucat in missing])
+    assert all(
+        [expanded_cats.get(ucat, None) is not None for ucat in unique_categories.keys()]
+    )
+
+    # Ensure that the queries are as expected
+    for cat_name, cat_query in unique_categories.items():
+        # Don't use get here because these fields should always exist
+        assert cat_query == expanded_cats[cat_name]["queries"]
+
+
+@pytest.mark.parametrize(
+    "options, call_counts, log_ind, expected_log_message",
+    [
+        (
+            {},
+            [8, 2, 2, 5],
+            2,
+            (
+                "\n!!!NOTE!!!\n You'll be able to find a performance comparison "
+                "here once the tests are complete (ensure you select the right framework): "
+                "https://treeherder.mozilla.org/perfherder/compare?originalProject=try&original"
+                "Revision=revision&newProject=try&newRevision=revision\n"
+            ),
+        ),
+        (
+            {"dry_run": True},
+            [8, 1, 1, 5],
+            2,
+            (
+                "\n!!!NOTE!!!\n You'll be able to find a performance comparison "
+                "here once the tests are complete (ensure you select the right framework): "
+                "https://treeherder.mozilla.org/perfherder/compare?originalProject=try&original"
+                "Revision=&newProject=try&newRevision=revision\n"
+            ),
+        ),
+        (
+            {"show_all": True},
+            [1, 2, 2, 3],
+            0,
+            (
+                "\n!!!NOTE!!!\n You'll be able to find a performance comparison "
+                "here once the tests are complete (ensure you select the right framework): "
+                "https://treeherder.mozilla.org/perfherder/compare?originalProject=try&original"
+                "Revision=revision&newProject=try&newRevision=revision\n"
+            ),
+        ),
+        (
+            {"single_run": True},
+            [8, 1, 1, 4],
+            2,
+            (
+                "If you need any help, you can find us in the #perf-help Matrix channel:\n"
+                "https://matrix.to/#/#perf-help:mozilla.org\n"
+            ),
+        ),
+    ],
+)
+@pytest.mark.skipif(os.name == "nt", reason="fzf not installed on host")
+def test_full_run(options, call_counts, log_ind, expected_log_message):
+    with mock.patch("tryselect.selectors.perf.push_to_try") as ptt, mock.patch(
+        "tryselect.selectors.perf.run_fzf"
+    ) as fzf, mock.patch(
+        "tryselect.selectors.perf.get_repository_object", new=mock.MagicMock()
+    ), mock.patch(
+        "tryselect.selectors.perf.LogProcessor.revision",
+        new_callable=mock.PropertyMock,
+        return_value="revision",
+    ) as logger, mock.patch(
+        "tryselect.selectors.perf.print",
+    ) as perf_print:
+        fzf.side_effect = [
+            ["", ["Benchmarks linux"]],
+            ["", TASKS],
+            ["", TASKS],
+            ["", TASKS],
+            ["", TASKS],
+            ["", TASKS],
+            ["", TASKS],
+            ["", TASKS],
+        ]
+
+        run(**options)
+
+        assert fzf.call_count == call_counts[0]
+        assert ptt.call_count == call_counts[1]
+        assert logger.call_count == call_counts[2]
+        assert perf_print.call_count == call_counts[3]
+        assert perf_print.call_args_list[log_ind][0][0] == expected_log_message
+
+
+@pytest.mark.parametrize(
+    "query, should_fail",
+    [
+        (
+            {
+                "query": {
+                    # Raptor has all variants available so it
+                    # should fail on this category
+                    "raptor": ["browsertime 'live 'no-fission"],
+                }
+            },
+            True,
+        ),
+        (
+            {
+                "query": {
+                    # Awsy has no variants defined so it shouldn't fail
+                    # on a query like this
+                    "awsy": ["browsertime 'live 'no-fission"],
+                }
+            },
+            False,
+        ),
+    ],
+)
+def test_category_rules(query, should_fail):
+    # Set the categories, and variants to expand
+    PerfParser.categories = {"test-live": query}
+    PerfParser.variants = TEST_VARIANTS
+
+    if should_fail:
+        with pytest.raises(InvalidCategoryException):
+            PerfParser.run_category_checks()
+    else:
+        assert PerfParser.run_category_checks()
+
+
+if __name__ == "__main__":
+    mozunit.main()