summaryrefslogtreecommitdiffstats
path: root/tools/tryselect
diff options
context:
space:
mode:
Diffstat (limited to 'tools/tryselect')
-rw-r--r--tools/tryselect/docs/img/treeherder-trigger-symbols.pngbin0 -> 44545 bytes
-rw-r--r--tools/tryselect/docs/index.rst35
-rw-r--r--tools/tryselect/docs/selectors/index.rst2
-rw-r--r--tools/tryselect/selectors/chooser/templates/chooser.html11
-rw-r--r--tools/tryselect/selectors/fuzzy.py7
-rw-r--r--tools/tryselect/selectors/perf.py23
-rw-r--r--tools/tryselect/task_config.py20
-rw-r--r--tools/tryselect/tasks.py11
-rw-r--r--tools/tryselect/test/test_fuzzy.py2
-rw-r--r--tools/tryselect/test/test_perf.py28
-rw-r--r--tools/tryselect/test/test_tasks.py98
-rw-r--r--tools/tryselect/try_presets.yml52
12 files changed, 281 insertions, 8 deletions
diff --git a/tools/tryselect/docs/img/treeherder-trigger-symbols.png b/tools/tryselect/docs/img/treeherder-trigger-symbols.png
new file mode 100644
index 0000000000..6c799c0bf0
--- /dev/null
+++ b/tools/tryselect/docs/img/treeherder-trigger-symbols.png
Binary files differ
diff --git a/tools/tryselect/docs/index.rst b/tools/tryselect/docs/index.rst
index 8978a72eaf..c42b7fbfb8 100644
--- a/tools/tryselect/docs/index.rst
+++ b/tools/tryselect/docs/index.rst
@@ -56,6 +56,41 @@ line instead, such as by using ``curl``:
.. _attach-job-review:
+Profiler symbols for try builds
+-------------------------------
+
+When profiling a tryserver build, you don't get symbols by default. You have to trigger
+an additional `upload-symbols` job on your try push so that the symbols are available
+on the symbol server.
+
+You can trigger this job manually in the Treeherder UI, using "Add new jobs (Search)...".
+
+Assuming you want to profile a "shippable" build (recommended), follow these steps:
+
+ 1. On the treeherder push, click the dropdown triangle in the top right corner.
+ 2. Select "Add new jobs (Search)..."
+ 3. Enter "shippable sym" in the search box and press enter.
+ 4. Important: Check the "Use full job list" checkbox.
+ 5. Pick the job for your try build. For Windows 64 bit builds, the job name is ``build-win64-shippable/opt-upload-symbols`` (this was written in February 2024).
+ 6. Click "Add selected", scroll down, and click "Trigger (1) Selected Jobs".
+
+Around ten minutes later, the symbols will be available on the symbol server, and profile symbolication will succeed.
+
+For other build types, choose the corresponding job for your build type. The job names all
+end in ``-upload-symbols``, and share a prefix with the build job.
+
+.. image:: img/treeherder-trigger-symbols.png
+
+If you've already captured a profile from a try build before the symbols were available, you can
+fix up the collected profile once the symbols are available. To do so, in the Firefox Profiler UI,
+click the "Profile Info" button in the top right corner, and then click the "Re-symbolicate profile"
+button in the panel.
+
+If you want to trigger the upload-symbol job when pushing to try, you can pick it in the list
+when running ``./mach try fuzzy --full`` - the ``--full`` part is necessary.
+The ``-upload-symbols`` task has a dependency on the build task, so you don't have to trigger
+the build task separately if you do this.
+
Adding Try jobs to a Phabricator patch
--------------------------------------
diff --git a/tools/tryselect/docs/selectors/index.rst b/tools/tryselect/docs/selectors/index.rst
index be618202d6..6aba6838d9 100644
--- a/tools/tryselect/docs/selectors/index.rst
+++ b/tools/tryselect/docs/selectors/index.rst
@@ -16,6 +16,7 @@ These are the currently implemented try selectors:
* :doc:`release <release>`: Prepare a tree for doing a staging release.
* :doc:`scriptworker <scriptworker>`: Run scriptworker tasks against a recent release.
* :doc:`compare <compare>`: Push two identical try jobs, one on your current commit and another of your choice
+* :ref:`perf <Mach Try Perf>`: Select categories of performance tests to run, and produce a before/after compare view link.
You can run them with:
@@ -42,3 +43,4 @@ See selector specific options by running:
Syntax <syntax>
Release <release>
Scriptworker <scriptworker>
+ Perf </testing/perfdocs/mach-try-perf>
diff --git a/tools/tryselect/selectors/chooser/templates/chooser.html b/tools/tryselect/selectors/chooser/templates/chooser.html
index 4e009d94ac..1c3df39327 100644
--- a/tools/tryselect/selectors/chooser/templates/chooser.html
+++ b/tools/tryselect/selectors/chooser/templates/chooser.html
@@ -40,6 +40,17 @@
{% else %}
<div class="tab-pane" id="{{ section.name }}" role="tabpanel" aria-labelledby="{{ section.name }}-tab">
{% endif %}
+ {% if section.name == "perf" %}
+ <div>
+ <p style="color:red">
+ WARNING: The try chooser is no longer supported for selecting performance tests.
+ Please use
+ <a href="https://firefox-source-docs.mozilla.org/testing/perfdocs/mach-try-perf.html">
+ ./mach try perf
+ </a>.
+ </p>
+ </div>
+ {% endif %}
{% for label, meta in section.labels|dictsort %}
<label class="multiselect filter-label" for={{ label }}>
<span>
diff --git a/tools/tryselect/selectors/fuzzy.py b/tools/tryselect/selectors/fuzzy.py
index 7a9bccc4b7..a9b139fa6f 100644
--- a/tools/tryselect/selectors/fuzzy.py
+++ b/tools/tryselect/selectors/fuzzy.py
@@ -12,7 +12,7 @@ from mach.util import get_state_dir
from ..cli import BaseTryParser
from ..push import check_working_directory, generate_try_task_config, push_to_try
-from ..tasks import filter_tasks_by_paths, generate_tasks
+from ..tasks import filter_tasks_by_paths, filter_tasks_by_worker_type, generate_tasks
from ..util.fzf import (
FZF_NOT_FOUND,
PREVIEW_SCRIPT,
@@ -182,6 +182,11 @@ def run(
if filter_by_uncommon_try_tasks(task_name)
}
+ if try_config_params.get("try_task_config", {}).get("worker-types", []):
+ all_tasks = filter_tasks_by_worker_type(all_tasks, try_config_params)
+ if not all_tasks:
+ return 1
+
if test_paths:
all_tasks = filter_tasks_by_paths(all_tasks, test_paths)
if not all_tasks:
diff --git a/tools/tryselect/selectors/perf.py b/tools/tryselect/selectors/perf.py
index 3c59e5949c..261905d5a3 100644
--- a/tools/tryselect/selectors/perf.py
+++ b/tools/tryselect/selectors/perf.py
@@ -336,6 +336,15 @@ class PerfParser(CompareParser):
"help": "Use PerfCompare Beta instead of CompareView.",
},
],
+ [
+ ["--non-pgo"],
+ {
+ "action": "store_true",
+ "default": False,
+ "help": "Use opt/non-pgo builds instead of shippable/pgo builds. "
+ "Setting this flag will result in faster try runs.",
+ },
+ ],
]
def get_tasks(base_cmd, queries, query_arg=None, candidate_tasks=None):
@@ -715,7 +724,7 @@ class PerfParser(CompareParser):
PerfParser.variants[variant.value]["query"]
)
- def _build_categories(category, category_info, category_matrix):
+ def _build_categories(category, category_info, category_matrix, **kwargs):
"""Builds the categories to display."""
categories = {}
@@ -745,6 +754,16 @@ class PerfParser(CompareParser):
for suite in category_info["suites"]
}
+ if kwargs.get("non_pgo"):
+ for key, query_list in platform_queries.items():
+ updated_query_list = []
+ for query in query_list:
+ updated_query = query.replace(
+ "'shippable", "!shippable !nightlyasrelease"
+ )
+ updated_query_list.append(updated_query)
+ platform_queries[key] = updated_query_list
+
platform_category_name = f"{category} {platform.value}"
platform_category_info = {
"queries": platform_queries,
@@ -931,7 +950,7 @@ class PerfParser(CompareParser):
for category, category_matrix in category_decision_matrix.items():
categories.update(
PerfParser._build_categories(
- category, PerfParser.categories[category], category_matrix
+ category, PerfParser.categories[category], category_matrix, **kwargs
)
)
diff --git a/tools/tryselect/task_config.py b/tools/tryselect/task_config.py
index f7a78cbfbf..58e8beeda4 100644
--- a/tools/tryselect/task_config.py
+++ b/tools/tryselect/task_config.py
@@ -80,7 +80,7 @@ class TryConfig(ParameterConfig):
def get_parameters(self, **kwargs):
result = self.try_config(**kwargs)
- if result is None:
+ if not result:
return None
return {"try_task_config": result}
@@ -579,9 +579,18 @@ class WorkerOverrides(TryConfig):
),
},
],
+ [
+ ["--worker-type"],
+ {
+ "action": "append",
+ "dest": "worker_types",
+ "default": [],
+ "help": "Select tasks that only run on the specified worker.",
+ },
+ ],
]
- def try_config(self, worker_overrides, worker_suffixes, **kwargs):
+ def try_config(self, worker_overrides, worker_suffixes, worker_types, **kwargs):
from gecko_taskgraph.util.workertypes import get_worker_type
from taskgraph.config import load_graph_config
@@ -621,8 +630,13 @@ class WorkerOverrides(TryConfig):
provisioner=provisioner, worker_type=worker_type, suffix=suffix
)
+ retVal = {}
+ if worker_types:
+ retVal["worker-types"] = list(overrides.keys()) + worker_types
+
if overrides:
- return {"worker-overrides": overrides}
+ retVal["worker-overrides"] = overrides
+ return retVal
all_task_configs = {
diff --git a/tools/tryselect/tasks.py b/tools/tryselect/tasks.py
index fa3eebc161..701d1c1ade 100644
--- a/tools/tryselect/tasks.py
+++ b/tools/tryselect/tasks.py
@@ -152,6 +152,17 @@ def generate_tasks(params=None, full=False, disable_target_task_filter=False):
return tg_target
+def filter_tasks_by_worker_type(tasks, params):
+ worker_types = params.get("try_task_config", {}).get("worker-types", [])
+ if worker_types:
+ retVal = {}
+ for t in tasks:
+ if tasks[t].task["workerType"] in worker_types:
+ retVal[t] = tasks[t]
+ return retVal
+ return tasks
+
+
def filter_tasks_by_paths(tasks, paths):
resolver = TestResolver.from_environment(cwd=here, loader_cls=TestManifestLoader)
run_suites, run_tests = resolver.resolve_metadata(paths)
diff --git a/tools/tryselect/test/test_fuzzy.py b/tools/tryselect/test/test_fuzzy.py
index 9ff1b386af..e68365c923 100644
--- a/tools/tryselect/test/test_fuzzy.py
+++ b/tools/tryselect/test/test_fuzzy.py
@@ -98,7 +98,7 @@ def test_query_paths_variants(run_mach, capfd, variant):
index = output.find(delim)
result = json.loads(output[index + len(delim) :])
tasks = result["parameters"]["try_task_config"]["tasks"]
- assert tasks == expected
+ assert sorted(tasks) == sorted(expected)
@pytest.mark.skipif(os.name == "nt", reason="fzf not installed on host")
diff --git a/tools/tryselect/test/test_perf.py b/tools/tryselect/test/test_perf.py
index 0db45df83e..c4e3b6b5fa 100644
--- a/tools/tryselect/test/test_perf.py
+++ b/tools/tryselect/test/test_perf.py
@@ -815,6 +815,34 @@ def test_category_expansion(
@pytest.mark.parametrize(
+ "category_options, call_counts",
+ [
+ (
+ {},
+ 0,
+ ),
+ (
+ {"non_pgo": True},
+ 58,
+ ),
+ ],
+)
+def test_category_expansion_with_non_pgo_flag(category_options, call_counts):
+ PerfParser.categories = TEST_CATEGORIES
+ PerfParser.variants = TEST_VARIANTS
+
+ expanded_cats = PerfParser.get_categories(**category_options)
+
+ non_shippable_count = 0
+ for cat_name in expanded_cats:
+ queries = str(expanded_cats[cat_name].get("queries", None))
+ if "!shippable !nightlyasrelease" in queries and "'shippable" not in queries:
+ non_shippable_count += 1
+
+ assert non_shippable_count == call_counts
+
+
+@pytest.mark.parametrize(
"options, call_counts, log_ind, expected_log_message",
[
(
diff --git a/tools/tryselect/test/test_tasks.py b/tools/tryselect/test/test_tasks.py
index 2e99c72d8b..ecf5849741 100644
--- a/tools/tryselect/test/test_tasks.py
+++ b/tools/tryselect/test/test_tasks.py
@@ -6,7 +6,103 @@ import os
import mozunit
import pytest
-from tryselect.tasks import cache_key, filter_tasks_by_paths, resolve_tests_by_suite
+from tryselect.tasks import (
+ cache_key,
+ filter_tasks_by_paths,
+ filter_tasks_by_worker_type,
+ resolve_tests_by_suite,
+)
+
+
+class task:
+ def __init__(self, workerType):
+ self.workerType = workerType
+
+ @property
+ def task(self):
+ return {"workerType": self.workerType}
+
+
+@pytest.mark.parametrize(
+ "tasks, params, expected",
+ (
+ pytest.param(
+ {
+ "foobar/xpcshell-1": task("t-unittest-314"),
+ "foobar/mochitest": task("t-unittest-157"),
+ "foobar/xpcshell-gpu": task("t-unittest-314-gpu"),
+ "foobar/xpcshell": task("t-unittest-314"),
+ },
+ {"try_task_config": {"worker-types": ["t-unittest-314"]}},
+ [
+ "foobar/xpcshell-1",
+ "foobar/xpcshell",
+ ],
+ id="single worker",
+ ),
+ pytest.param(
+ {
+ "foobar/xpcshell-1": task("t-unittest-314"),
+ "foobar/mochitest": task("t-unittest-157"),
+ "foobar/xpcshell-gpu": task("t-unittest-314-gpu"),
+ "foobar/xpcshell": task("t-unittest-314"),
+ },
+ {
+ "try_task_config": {
+ "worker-types": ["t-unittest-314", "t-unittest-314-gpu"]
+ }
+ },
+ [
+ "foobar/xpcshell-1",
+ "foobar/xpcshell-gpu",
+ "foobar/xpcshell",
+ ],
+ id="multiple workers worker",
+ ),
+ pytest.param(
+ {
+ "foobar/xpcshell-1": task("t-unittest-314"),
+ "foobar/mochitest": task("t-unittest-157"),
+ "foobar/xpcshell-gpu": task("t-unittest-314-gpu"),
+ "foobar/xpcshell": task("t-unittest-314"),
+ },
+ {"try_task_config": {"worker-types": ["t-unittest-157"]}},
+ [
+ "foobar/mochitest",
+ ],
+ id="single task",
+ ),
+ pytest.param(
+ {
+ "foobar/xpcshell-1": task("t-unittest-314"),
+ "foobar/mochitest": task("t-unittest-157"),
+ "foobar/xpcshell-gpu": task("t-unittest-314-gpu"),
+ "foobar/xpcshell": task("t-unittest-314"),
+ },
+ {"try_task_config": {"worker-types": []}},
+ [
+ "foobar/xpcshell-1",
+ "foobar/mochitest",
+ "foobar/xpcshell-gpu",
+ "foobar/xpcshell",
+ ],
+ id="no worker",
+ ),
+ pytest.param(
+ {
+ "foobar/xpcshell-1": task("t-unittest-314"),
+ "foobar/mochitest": task("t-unittest-157"),
+ "foobar/xpcshell-gpu": task("t-unittest-314-gpu"),
+ "foobar/xpcshell": task("t-unittest-314"),
+ },
+ {"try_task_config": {"worker-types": ["fake-worker"]}},
+ [],
+ id="invalid worker",
+ ),
+ ),
+)
+def test_filter_tasks_by_worker_type(patch_resolver, tasks, params, expected):
+ assert list(filter_tasks_by_worker_type(tasks, params)) == expected
def test_filter_tasks_by_paths(patch_resolver):
diff --git a/tools/tryselect/try_presets.yml b/tools/tryselect/try_presets.yml
index ebdc94aa03..680d318dc6 100644
--- a/tools/tryselect/try_presets.yml
+++ b/tools/tryselect/try_presets.yml
@@ -9,6 +9,14 @@
# yamllint disable rule:line-length
+android-components:
+ selector: fuzzy
+ description: >-
+ Run android-components builds and tests.
+ query:
+ - "'build-components"
+ - "'test-components"
+
builds:
selector: fuzzy
description: >-
@@ -83,6 +91,50 @@ devtools-linux:
- "'mochitest-devtools-chrome | 'mochitest-chrome-1proc 'linux '64-qr/ !swr"
- "'xpcshell 'linux !nofis '64-qr/"
+fenix:
+ selector: fuzzy
+ description: >-
+ Run Fenix builds and tests.
+ query:
+ - "'build-apk-fenix-debug"
+ - "'signing-apk-fenix-debug"
+ - "'build-apk-fenix-android-test-debug"
+ - "'signing-apk-fenix-android-test-debug"
+ - "'test-apk-fenix-debug"
+ - "'ui-test-apk-fenix-arm-debug"
+
+firefox-android:
+ selector: fuzzy
+ description: >-
+ Run android-components, Focus, and Fenix builds and tests.
+ query:
+ - "'build-components"
+ - "'test-components"
+ - "'build-apk-fenix-debug"
+ - "'signing-apk-fenix-debug"
+ - "'build-apk-fenix-android-test-debug"
+ - "'signing-apk-fenix-android-test-debug"
+ - "'test-apk-fenix-debug"
+ - "'ui-test-apk-fenix-arm-debug"
+ - "'build-apk-focus-debug"
+ - "'signing-apk-focus-debug"
+ - "'build-apk-focus-android-test-debug"
+ - "'signing-apk-focus-android-test-debug"
+ - "'test-apk-focus-debug"
+ - "'ui-test-apk-focus-arm-debug"
+
+focus:
+ selector: fuzzy
+ description: >-
+ Run Focus builds and tests.
+ query:
+ - "'build-apk-focus-debug"
+ - "'signing-apk-focus-debug"
+ - "'build-apk-focus-android-test-debug"
+ - "'signing-apk-focus-android-test-debug"
+ - "'test-apk-focus-debug"
+ - "'ui-test-apk-focus-arm-debug"
+
fpush-linux-android:
selector: fuzzy
description: >-
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-20 15:20:18 +0000