1
0
Fork 0
firefox/taskcluster/gecko_taskgraph/test/test_optimize_strategies.py
Daniel Baumann 5e9a113729
Adding upstream version 140.0.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
2025-06-25 09:37:52 +02:00

673 lines
20 KiB
Python

# Any copyright is dedicated to the public domain.
# http://creativecommons.org/publicdomain/zero/1.0/
import time
from datetime import datetime
from time import mktime
import pytest
from mozunit import main
from taskgraph.optimize.base import registry
from taskgraph.task import Task
from taskgraph.util.copy import deepcopy
from gecko_taskgraph.optimize import project
from gecko_taskgraph.optimize.backstop import SkipUnlessBackstop, SkipUnlessPushInterval
from gecko_taskgraph.optimize.bugbug import (
FALLBACK,
BugBugPushSchedules,
DisperseGroups,
SkipUnlessDebug,
)
from gecko_taskgraph.optimize.mozlint import SkipUnlessMozlint
from gecko_taskgraph.optimize.strategies import SkipUnlessMissing, SkipUnlessSchedules
from gecko_taskgraph.util.backstop import BACKSTOP_PUSH_INTERVAL
from gecko_taskgraph.util.bugbug import (
BUGBUG_BASE_URL,
BugbugTimeoutException,
push_schedules,
)
@pytest.fixture(autouse=True)
def clear_push_schedules_memoize():
push_schedules.clear()
@pytest.fixture
def params():
return {
"branch": "autoland",
"head_repository": "https://hg.mozilla.org/integration/autoland",
"head_rev": "abcdef",
"project": "autoland",
"pushlog_id": 1,
"pushdate": mktime(datetime.now().timetuple()),
}
def generate_tasks(*tasks):
for i, task in enumerate(tasks):
task.setdefault("label", f"task-{i}-label")
task.setdefault("kind", "test")
task.setdefault("task", {})
task.setdefault("attributes", {})
task["attributes"].setdefault("e10s", True)
for attr in (
"optimization",
"dependencies",
"soft_dependencies",
):
task.setdefault(attr, None)
task["task"].setdefault("label", task["label"])
yield Task.from_json(task)
# task sets
default_tasks = list(
generate_tasks(
{"attributes": {"test_manifests": ["foo/test.ini", "bar/test.ini"]}},
{"attributes": {"test_manifests": ["bar/test.ini"], "build_type": "debug"}},
{"attributes": {"build_type": "debug"}},
{"attributes": {"test_manifests": [], "build_type": "opt"}},
{"attributes": {"build_type": "opt"}},
)
)
disperse_tasks = list(
generate_tasks(
{
"attributes": {
"test_manifests": ["foo/test.ini", "bar/test.ini"],
"test_platform": "linux/opt",
}
},
{
"attributes": {
"test_manifests": ["bar/test.ini"],
"test_platform": "linux/opt",
}
},
{
"attributes": {
"test_manifests": ["bar/test.ini"],
"test_platform": "windows/debug",
}
},
{
"attributes": {
"test_manifests": ["bar/test.ini"],
"test_platform": "linux/opt",
"unittest_variant": "no-fission",
}
},
{
"attributes": {
"e10s": False,
"test_manifests": ["bar/test.ini"],
"test_platform": "linux/opt",
}
},
)
)
def idfn(param):
if isinstance(param, tuple):
try:
return param[0].__name__
except AttributeError:
return None
return None
@pytest.mark.parametrize(
"opt,tasks,arg,expected",
[
# debug
pytest.param(
SkipUnlessDebug(),
default_tasks,
None,
["task-0-label", "task-1-label", "task-2-label"],
),
# disperse with no supplied importance
pytest.param(
DisperseGroups(),
disperse_tasks,
None,
[t.label for t in disperse_tasks],
),
# disperse with low importance
pytest.param(
DisperseGroups(),
disperse_tasks,
{"bar/test.ini": "low"},
["task-0-label", "task-2-label"],
),
# disperse with medium importance
pytest.param(
DisperseGroups(),
disperse_tasks,
{"bar/test.ini": "medium"},
["task-0-label", "task-1-label", "task-2-label"],
),
# disperse with high importance
pytest.param(
DisperseGroups(),
disperse_tasks,
{"bar/test.ini": "high"},
["task-0-label", "task-1-label", "task-2-label", "task-3-label"],
),
],
ids=idfn,
)
def test_optimization_strategy_remove(params, opt, tasks, arg, expected):
labels = [t.label for t in tasks if not opt.should_remove_task(t, params, arg)]
assert sorted(labels) == sorted(expected)
@pytest.mark.parametrize(
"args,data,expected",
[
# empty
pytest.param(
(0.1,),
{},
[],
),
# only tasks without test manifests selected
pytest.param(
(0.1,),
{"tasks": {"task-1-label": 0.9, "task-2-label": 0.1, "task-3-label": 0.5}},
["task-2-label"],
),
# tasks which are unknown to bugbug are selected
pytest.param(
(0.1,),
{
"tasks": {"task-1-label": 0.9, "task-3-label": 0.5},
"known_tasks": ["task-1-label", "task-3-label", "task-4-label"],
},
["task-2-label"],
),
# tasks containing groups selected
pytest.param(
(0.1,),
{"groups": {"foo/test.ini": 0.4}},
["task-0-label"],
),
# tasks matching "tasks" or "groups" selected
pytest.param(
(0.1,),
{
"tasks": {"task-2-label": 0.2},
"groups": {"foo/test.ini": 0.25, "bar/test.ini": 0.75},
},
["task-0-label", "task-1-label", "task-2-label"],
),
# tasks matching "tasks" or "groups" selected, when they exceed the confidence threshold
pytest.param(
(0.5,),
{
"tasks": {"task-2-label": 0.2, "task-4-label": 0.5},
"groups": {"foo/test.ini": 0.65, "bar/test.ini": 0.25},
},
["task-0-label", "task-4-label"],
),
# tasks matching "reduced_tasks" are selected, when they exceed the confidence threshold
pytest.param(
(0.7, True, True),
{
"tasks": {"task-2-label": 0.7, "task-4-label": 0.7},
"reduced_tasks": {"task-4-label": 0.7},
"groups": {"foo/test.ini": 0.75, "bar/test.ini": 0.25},
},
["task-4-label"],
),
# tasks matching "groups" selected, only on specific platforms.
pytest.param(
(0.1, False, False, None, 1, True),
{
"tasks": {"task-2-label": 0.2},
"groups": {"foo/test.ini": 0.25, "bar/test.ini": 0.75},
"config_groups": {
"foo/test.ini": ["task-1-label", "task-0-label"],
"bar/test.ini": ["task-0-label"],
},
},
["task-0-label", "task-2-label"],
),
pytest.param(
(0.1, False, False, None, 1, True),
{
"tasks": {"task-2-label": 0.2},
"groups": {"foo/test.ini": 0.25, "bar/test.ini": 0.75},
"config_groups": {
"foo/test.ini": ["task-1-label", "task-0-label"],
"bar/test.ini": ["task-1-label"],
},
},
["task-0-label", "task-1-label", "task-2-label"],
),
pytest.param(
(0.1, False, False, None, 1, True),
{
"tasks": {"task-2-label": 0.2},
"groups": {"foo/test.ini": 0.25, "bar/test.ini": 0.75},
"config_groups": {
"foo/test.ini": ["task-1-label"],
"bar/test.ini": ["task-0-label"],
},
},
["task-0-label", "task-2-label"],
),
pytest.param(
(0.1, False, False, None, 1, True),
{
"tasks": {"task-2-label": 0.2},
"groups": {"foo/test.ini": 0.25, "bar/test.ini": 0.75},
"config_groups": {
"foo/test.ini": ["task-1-label"],
"bar/test.ini": ["task-3-label"],
},
},
["task-2-label"],
),
],
ids=idfn,
)
def test_bugbug_push_schedules(responses, params, args, data, expected):
query = "/push/{branch}/{head_rev}/schedules".format(**params)
url = BUGBUG_BASE_URL + query
responses.add(
responses.GET,
url,
json=data,
status=200,
)
opt = BugBugPushSchedules(*args)
labels = [
t.label for t in default_tasks if not opt.should_remove_task(t, params, {})
]
assert sorted(labels) == sorted(expected)
def test_bugbug_multiple_pushes(responses, params):
pushes = {str(pid): {"changesets": [f"c{pid}"]} for pid in range(8, 10)}
responses.add(
responses.GET,
"https://hg.mozilla.org/integration/autoland/json-pushes/?version=2&startID=8&endID=9",
json={"pushes": pushes},
status=200,
)
responses.add(
responses.GET,
BUGBUG_BASE_URL + "/push/{}/c9/schedules".format(params["branch"]),
json={
"tasks": {"task-2-label": 0.2, "task-4-label": 0.5},
"groups": {"foo/test.ini": 0.2, "bar/test.ini": 0.25},
"config_groups": {"foo/test.ini": ["linux-*"], "bar/test.ini": ["task-*"]},
"known_tasks": ["task-4-label"],
},
status=200,
)
# Tasks with a lower confidence don't override task with a higher one.
# Tasks with a higher confidence override tasks with a lower one.
# Known tasks are merged.
responses.add(
responses.GET,
BUGBUG_BASE_URL + "/push/{branch}/{head_rev}/schedules".format(**params),
json={
"tasks": {"task-2-label": 0.2, "task-4-label": 0.2},
"groups": {"foo/test.ini": 0.65, "bar/test.ini": 0.25},
"config_groups": {
"foo/test.ini": ["task-*"],
"bar/test.ini": ["windows-*"],
},
"known_tasks": ["task-1-label", "task-3-label"],
},
status=200,
)
params["pushlog_id"] = 10
opt = BugBugPushSchedules(0.3, False, False, False, 2)
labels = [
t.label for t in default_tasks if not opt.should_remove_task(t, params, {})
]
assert sorted(labels) == sorted(["task-0-label", "task-2-label", "task-4-label"])
opt = BugBugPushSchedules(0.3, False, False, False, 2, True)
labels = [
t.label for t in default_tasks if not opt.should_remove_task(t, params, {})
]
assert sorted(labels) == sorted(["task-0-label", "task-2-label", "task-4-label"])
opt = BugBugPushSchedules(0.2, False, False, False, 2, True)
labels = [
t.label for t in default_tasks if not opt.should_remove_task(t, params, {})
]
assert sorted(labels) == sorted(
["task-0-label", "task-1-label", "task-2-label", "task-4-label"]
)
def test_bugbug_timeout(monkeypatch, responses, params):
query = "/push/{branch}/{head_rev}/schedules".format(**params)
url = BUGBUG_BASE_URL + query
responses.add(
responses.GET,
url,
json={"ready": False},
status=202,
)
# Make sure the test runs fast.
monkeypatch.setattr(time, "sleep", lambda i: None)
opt = BugBugPushSchedules(0.5)
with pytest.raises(BugbugTimeoutException):
opt.should_remove_task(default_tasks[0], params, None)
def test_bugbug_fallback(monkeypatch, responses, params):
query = "/push/{branch}/{head_rev}/schedules".format(**params)
url = BUGBUG_BASE_URL + query
responses.add(
responses.GET,
url,
json={"ready": False},
status=202,
)
opt = BugBugPushSchedules(0.5, fallback=FALLBACK)
# Make sure the test runs fast.
monkeypatch.setattr(time, "sleep", lambda i: None)
def fake_should_remove_task(task, params, _):
return task.label == default_tasks[0].label
monkeypatch.setattr(
registry[FALLBACK], "should_remove_task", fake_should_remove_task
)
assert opt.should_remove_task(default_tasks[0], params, None)
# Make sure we don't hit bugbug more than once.
responses.reset()
assert not opt.should_remove_task(default_tasks[1], params, None)
def test_backstop(params):
all_labels = {t.label for t in default_tasks}
opt = SkipUnlessBackstop()
params["backstop"] = False
scheduled = {
t.label for t in default_tasks if not opt.should_remove_task(t, params, None)
}
assert scheduled == set()
params["backstop"] = True
scheduled = {
t.label for t in default_tasks if not opt.should_remove_task(t, params, None)
}
assert scheduled == all_labels
def test_push_interval(params):
all_labels = {t.label for t in default_tasks}
opt = SkipUnlessPushInterval(10) # every 10th push
# Only multiples of 10 schedule tasks.
params["pushlog_id"] = 9
scheduled = {
t.label for t in default_tasks if not opt.should_remove_task(t, params, None)
}
assert scheduled == set()
params["pushlog_id"] = 10
scheduled = {
t.label for t in default_tasks if not opt.should_remove_task(t, params, None)
}
assert scheduled == all_labels
def test_expanded(params):
all_labels = {t.label for t in default_tasks}
opt = registry["skip-unless-expanded"]
params["backstop"] = False
params["pushlog_id"] = BACKSTOP_PUSH_INTERVAL / 2
scheduled = {
t.label for t in default_tasks if not opt.should_remove_task(t, params, None)
}
assert scheduled == all_labels
params["pushlog_id"] += 1
scheduled = {
t.label for t in default_tasks if not opt.should_remove_task(t, params, None)
}
assert scheduled == set()
params["backstop"] = True
scheduled = {
t.label for t in default_tasks if not opt.should_remove_task(t, params, None)
}
assert scheduled == all_labels
def test_project_autoland_test(monkeypatch, responses, params):
"""Tests the behaviour of the `project.autoland["test"]` strategy on
various types of pushes.
"""
# This is meant to test the composition of substrategies, and not the
# actual optimization implementations. So mock them out for simplicity.
monkeypatch.setattr(SkipUnlessSchedules, "should_remove_task", lambda *args: False)
monkeypatch.setattr(DisperseGroups, "should_remove_task", lambda *args: False)
def fake_bugbug_should_remove_task(self, task, params, importance):
if self.num_pushes > 1:
return task.label == "task-4-label"
return task.label in ("task-2-label", "task-3-label", "task-4-label")
monkeypatch.setattr(
BugBugPushSchedules, "should_remove_task", fake_bugbug_should_remove_task
)
opt = project.autoland["test"]
# On backstop pushes, nothing gets optimized.
params["backstop"] = True
scheduled = {
t.label for t in default_tasks if not opt.should_remove_task(t, params, {})
}
assert scheduled == {t.label for t in default_tasks}
# On expanded pushes, some things are optimized.
params["backstop"] = False
params["pushlog_id"] = 10
scheduled = {
t.label for t in default_tasks if not opt.should_remove_task(t, params, {})
}
assert scheduled == {"task-0-label", "task-1-label", "task-2-label", "task-3-label"}
# On regular pushes, more things are optimized.
params["pushlog_id"] = 11
scheduled = {
t.label for t in default_tasks if not opt.should_remove_task(t, params, {})
}
assert scheduled == {"task-0-label", "task-1-label"}
@pytest.mark.parametrize(
"pushed_files,to_lint,expected",
[
pytest.param(
["a/b/c.txt"],
[],
True,
),
pytest.param(
["python/mozlint/a/support_file.txt", "b/c/d.txt"],
["python/mozlint/a/support_file.txt"],
False,
),
],
ids=idfn,
)
def test_mozlint_should_remove_task(
monkeypatch, params, pushed_files, to_lint, expected
):
import mozlint.pathutils
class MockParser:
def __call__(self, *args, **kwargs):
return []
def mock_filterpaths(*args, **kwargs):
return to_lint, None
monkeypatch.setattr(mozlint.pathutils, "filterpaths", mock_filterpaths)
opt = SkipUnlessMozlint("")
monkeypatch.setattr(opt, "mozlint_parser", MockParser())
params["files_changed"] = pushed_files
result = opt.should_remove_task(default_tasks[0], params, "")
assert result == expected
@pytest.mark.parametrize(
"pushed_files,linter_config,expected",
[
pytest.param(
["a/b/c.txt"],
[{"include": ["b/c"]}],
True,
),
pytest.param(
["a/b/c.txt"],
[{"include": ["a/b"], "exclude": ["a/b/c.txt"]}],
True,
),
pytest.param(
["python/mozlint/a/support_file.txt", "b/c/d.txt"],
[{}],
False,
),
],
ids=idfn,
)
def test_mozlint_should_remove_task2(
monkeypatch, params, pushed_files, linter_config, expected
):
class MockParser:
def __call__(self, *args, **kwargs):
return linter_config
opt = SkipUnlessMozlint("")
monkeypatch.setattr(opt, "mozlint_parser", MockParser())
params["files_changed"] = pushed_files
result = opt.should_remove_task(default_tasks[0], params, "")
assert result == expected
def test_skip_unless_missing(responses, params):
opt = SkipUnlessMissing()
task = deepcopy(default_tasks[0])
task.task["deadline"] = "2024-01-02T00:00:00.000Z"
index = "foo.bar.baz"
task_id = "abc"
root_url = "https://firefox-ci-tc.services.mozilla.com/api"
# Task is missing, don't optimize
responses.add(
responses.GET,
f"{root_url}/index/v1/task/{index}",
status=404,
)
result = opt.should_remove_task(task, params, index)
assert result is False
# Task is found but failed, don't optimize
responses.replace(
responses.GET,
f"{root_url}/index/v1/task/{index}",
json={"taskId": task_id},
status=200,
)
responses.add(
responses.GET,
f"{root_url}/queue/v1/task/{task_id}/status",
json={"status": {"state": "failed"}},
status=200,
)
result = opt.should_remove_task(task, params, index)
assert result is False
# Task is found and passed but expires before deadline, don't optimize
responses.replace(
responses.GET,
f"{root_url}/index/v1/task/{index}",
json={"taskId": task_id},
status=200,
)
responses.replace(
responses.GET,
f"{root_url}/queue/v1/task/{task_id}/status",
json={"status": {"state": "completed", "expires": "2024-01-01T00:00:00.000Z"}},
status=200,
)
result = opt.should_remove_task(task, params, index)
assert result is False
# Task is found and passed and expires after deadline, optimize
responses.replace(
responses.GET,
f"{root_url}/index/v1/task/{index}",
json={"taskId": task_id},
status=200,
)
responses.replace(
responses.GET,
f"{root_url}/queue/v1/task/{task_id}/status",
json={"status": {"state": "completed", "expires": "2024-01-03T00:00:00.000Z"}},
status=200,
)
result = opt.should_remove_task(task, params, index)
assert result is True
# Task has parameterized deadline, does not raise
task.task["deadline"] = {"relative-datestamp": "1 day"}
responses.replace(
responses.GET,
f"{root_url}/index/v1/task/{index}",
json={"taskId": task_id},
status=200,
)
responses.replace(
responses.GET,
f"{root_url}/queue/v1/task/{task_id}/status",
json={"status": {"state": "completed", "expires": "2024-01-03T00:00:00.000Z"}},
status=200,
)
opt.should_remove_task(task, params, index)
if __name__ == "__main__":
main()