134 lines
4.9 KiB
Python
134 lines
4.9 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
|
|
import datetime
|
|
import logging
|
|
|
|
import mozpack.path as mozpath
|
|
from mozbuild.base import MozbuildObject
|
|
from mozbuild.util import memoize
|
|
from taskgraph.optimize.base import OptimizationStrategy, register_strategy
|
|
from taskgraph.optimize.strategies import IndexSearch
|
|
from taskgraph.util.parameterization import resolve_timestamps
|
|
from taskgraph.util.path import match as match_path
|
|
|
|
from gecko_taskgraph.optimize.mozlint import SkipUnlessMozlint
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@register_strategy("skip-unless-schedules")
|
|
class SkipUnlessSchedules(OptimizationStrategy):
|
|
@memoize
|
|
def scheduled_by_push(self, files_changed):
|
|
mbo = MozbuildObject.from_environment()
|
|
# the decision task has a sparse checkout, so, mozbuild_reader will use
|
|
# a MercurialRevisionFinder with revision '.', which should be the same
|
|
# as `revision`; in other circumstances, it will use a default reader
|
|
rdr = mbo.mozbuild_reader(config_mode="empty")
|
|
|
|
components = set()
|
|
for p, m in rdr.files_info(files_changed).items():
|
|
components |= set(m["SCHEDULES"].components)
|
|
|
|
return components
|
|
|
|
def should_remove_task(self, task, params, conditions):
|
|
if params.get("pushlog_id") == -1:
|
|
return False
|
|
|
|
scheduled = self.scheduled_by_push(frozenset(params["files_changed"]))
|
|
conditions = set(conditions)
|
|
# if *any* of the condition components are scheduled, do not optimize
|
|
if conditions & scheduled:
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
@register_strategy("skip-unless-has-relevant-tests")
|
|
class SkipUnlessHasRelevantTests(OptimizationStrategy):
|
|
"""Optimizes tasks that don't run any tests that were
|
|
in child directories of a modified file.
|
|
"""
|
|
|
|
@memoize
|
|
def get_changed_dirs(self, files_changed):
|
|
changed = map(mozpath.dirname, files_changed)
|
|
# Filter out empty directories (from files modified in the root).
|
|
# Otherwise all tasks would be scheduled.
|
|
return {d for d in changed if d}
|
|
|
|
def should_remove_task(self, task, params, _):
|
|
if not task.attributes.get("test_manifests"):
|
|
return True
|
|
|
|
for d in self.get_changed_dirs(frozenset(params["files_changed"])):
|
|
for t in task.attributes["test_manifests"]:
|
|
if t.startswith(d):
|
|
logger.debug(
|
|
f"{task.label} runs a test path ({t}) contained by a modified file ({d})"
|
|
)
|
|
return False
|
|
return True
|
|
|
|
|
|
# TODO: This overwrites upstream Taskgraph's `skip-unless-changed`
|
|
# optimization. Once the firefox-android migration is landed and we upgrade
|
|
# upstream Taskgraph to a version that doesn't call files_changed.check`, this
|
|
# class can be deleted. Also remove the `taskgraph.optimize.base.registry` tweak
|
|
# in `gecko_taskgraph.register` at the same time.
|
|
@register_strategy("skip-unless-changed")
|
|
class SkipUnlessChanged(OptimizationStrategy):
|
|
def check(self, files_changed, patterns):
|
|
for pattern in patterns:
|
|
for path in files_changed:
|
|
if match_path(path, pattern):
|
|
return True
|
|
return False
|
|
|
|
def should_remove_task(self, task, params, file_patterns):
|
|
# pushlog_id == -1 - this is the case when run from a cron.yml job or on a git repository
|
|
if params.get("repository_type") == "hg" and params.get("pushlog_id") == -1:
|
|
return False
|
|
|
|
changed = self.check(params["files_changed"], file_patterns)
|
|
if not changed:
|
|
logger.debug(
|
|
f'no files found matching a pattern in `skip-unless-changed` for "{task.label}"'
|
|
)
|
|
return True
|
|
return False
|
|
|
|
|
|
register_strategy("skip-unless-mozlint", args=("tools/lint",))(SkipUnlessMozlint)
|
|
|
|
|
|
@register_strategy("skip-unless-missing")
|
|
class SkipUnlessMissing(OptimizationStrategy):
|
|
"""Skips a task unless it is missing from a specified index.
|
|
|
|
This simply defers to Taskgraph's `index-search` optimization. The reason
|
|
we need this shim is because replacement and removal optimizations can't be
|
|
joined together in a composite strategy as removal and replacement happen
|
|
at different times.
|
|
"""
|
|
|
|
index_search = IndexSearch()
|
|
|
|
def _convert_datetime_str(self, dt):
|
|
if dt.endswith("Z"):
|
|
dt = dt[:-1]
|
|
|
|
return datetime.datetime.fromisoformat(dt).strftime(self.index_search.fmt)
|
|
|
|
def should_remove_task(self, task, params, index):
|
|
now = datetime.datetime.now(datetime.timezone.utc)
|
|
deadline = self._convert_datetime_str(
|
|
resolve_timestamps(now, task.task["deadline"])
|
|
)
|
|
return bool(
|
|
self.index_search.should_replace_task(task, params, deadline, [index])
|
|
)
|