646 lines
21 KiB
Python
646 lines
21 KiB
Python
# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
|
|
# vim: set filetype=python:
|
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
import glob
|
|
import itertools
|
|
import json
|
|
import os
|
|
import re
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
import xml.etree.ElementTree as ET
|
|
|
|
import mozpack.path as mozpath
|
|
from mozlint import result
|
|
from mozpack.files import FileFinder
|
|
|
|
# The Gradle target invocations are serialized with a simple locking file scheme. It's fine for
|
|
# them to take a while, since the first will compile all the Java, etc, and then perform
|
|
# potentially expensive static analyses.
|
|
GRADLE_LOCK_MAX_WAIT_SECONDS = 20 * 60
|
|
|
|
EXCLUSION_FILES = [
|
|
os.path.join("tools", "rewriting", "Generated.txt"),
|
|
os.path.join("tools", "rewriting", "ThirdPartyPaths.txt"),
|
|
]
|
|
|
|
|
|
def setup(root, **setupargs):
|
|
if setupargs.get("substs", {}).get("MOZ_BUILD_APP") != "mobile/android":
|
|
return -1
|
|
|
|
if "topobjdir" not in setupargs:
|
|
setupargs["log"].debug(
|
|
f"Skipping {setupargs['name']}: a configured Android build is required!"
|
|
)
|
|
return -1
|
|
|
|
return 0
|
|
|
|
|
|
def gradle(log, topsrcdir=None, topobjdir=None, tasks=[], extra_args=[], verbose=True):
|
|
sys.path.insert(0, os.path.join(topsrcdir, "mobile", "android"))
|
|
from gradle import gradle_lock
|
|
|
|
with gradle_lock(topobjdir, max_wait_seconds=GRADLE_LOCK_MAX_WAIT_SECONDS):
|
|
# The android-lint parameter can be used by gradle tasks to run special
|
|
# logic when they are run for a lint using
|
|
# project.hasProperty('android-lint')
|
|
cmd_args = (
|
|
[
|
|
sys.executable,
|
|
os.path.join(topsrcdir, "mach"),
|
|
"gradle",
|
|
"--verbose",
|
|
"-Pandroid-lint",
|
|
"--",
|
|
]
|
|
+ tasks
|
|
+ extra_args
|
|
)
|
|
|
|
cmd = " ".join(shlex.quote(arg) for arg in cmd_args)
|
|
log.debug(cmd)
|
|
|
|
# Gradle and mozprocess do not get along well, so we use subprocess
|
|
# directly.
|
|
proc = subprocess.Popen(cmd_args, cwd=topsrcdir)
|
|
status = None
|
|
# Leave it to the subprocess to handle Ctrl+C. If it terminates as a result
|
|
# of Ctrl+C, proc.wait() will return a status code, and, we get out of the
|
|
# loop. If it doesn't, like e.g. gdb, we continue waiting.
|
|
while status is None:
|
|
try:
|
|
status = proc.wait()
|
|
except KeyboardInterrupt:
|
|
pass
|
|
|
|
try:
|
|
proc.wait()
|
|
except KeyboardInterrupt:
|
|
proc.kill()
|
|
raise
|
|
|
|
return proc.returncode
|
|
|
|
|
|
def format(_paths, config, fix=None, **lintargs):
|
|
topsrcdir = lintargs["root"]
|
|
topobjdir = lintargs["topobjdir"]
|
|
|
|
if fix:
|
|
tasks = lintargs["substs"]["GRADLE_ANDROID_FORMAT_LINT_FIX_TASKS"]
|
|
else:
|
|
tasks = lintargs["substs"]["GRADLE_ANDROID_FORMAT_LINT_CHECK_TASKS"]
|
|
|
|
ret = gradle(
|
|
lintargs["log"],
|
|
topsrcdir=topsrcdir,
|
|
topobjdir=topobjdir,
|
|
tasks=tasks,
|
|
extra_args=lintargs.get("extra_args") or [],
|
|
)
|
|
|
|
results = []
|
|
for path in lintargs["substs"]["GRADLE_ANDROID_FORMAT_LINT_FOLDERS"]:
|
|
for outdir in ("spotless-clean", "spotless-lints"):
|
|
folder = os.path.join(
|
|
topobjdir, "gradle", "build", path, outdir, "spotlessJava"
|
|
)
|
|
for filename in glob.iglob(folder + "/**/*.java", recursive=True):
|
|
err = {
|
|
"rule": "spotless-java",
|
|
"path": os.path.join(
|
|
topsrcdir, path, mozpath.relpath(filename, folder)
|
|
),
|
|
"lineno": 0,
|
|
"column": 0,
|
|
"message": "Formatting error, please run ./mach lint -l android-format --fix",
|
|
"level": "error",
|
|
}
|
|
results.append(result.from_config(config, **err))
|
|
folder = os.path.join(
|
|
topobjdir, "gradle", "build", path, outdir, "spotlessKotlin"
|
|
)
|
|
for filename in glob.iglob(folder + "/**/*.kt", recursive=True):
|
|
err = {
|
|
"rule": "spotless-kt",
|
|
"path": os.path.join(
|
|
topsrcdir, path, mozpath.relpath(filename, folder)
|
|
),
|
|
"lineno": 0,
|
|
"column": 0,
|
|
"message": "Formatting error, please run ./mach lint -l android-format --fix",
|
|
"level": "error",
|
|
}
|
|
results.append(result.from_config(config, **err))
|
|
|
|
if len(results) == 0 and ret != 0:
|
|
# spotless seems to hit unfixed error.
|
|
err = {
|
|
"rule": "spotless",
|
|
"path": "",
|
|
"lineno": 0,
|
|
"column": 0,
|
|
"message": "Unexpected error",
|
|
"level": "error",
|
|
}
|
|
results.append(result.from_config(config, **err))
|
|
|
|
# If --fix was passed, we just report the number of files that were changed
|
|
if fix:
|
|
return {"results": [], "fixed": len(results)}
|
|
return results
|
|
|
|
|
|
def fenix_format(_paths, config, fix=None, **lintargs):
|
|
return report_gradlew(
|
|
config,
|
|
fix,
|
|
os.path.join("mobile", "android", "fenix"),
|
|
lint_tasks=[
|
|
"fenix:lint",
|
|
"fenix:lintFenixDebug",
|
|
],
|
|
**lintargs,
|
|
)
|
|
|
|
|
|
def ac_format(_paths, config, fix=None, **lintargs):
|
|
return report_gradlew(
|
|
config,
|
|
fix,
|
|
os.path.join("mobile", "android", "android-components"),
|
|
lint_tasks=["lint-a-c"],
|
|
**lintargs,
|
|
)
|
|
|
|
|
|
def focus_format(_paths, config, fix=None, **lintargs):
|
|
return report_gradlew(
|
|
config,
|
|
fix,
|
|
os.path.join("mobile", "android", "focus-android"),
|
|
lint_tasks=["focus-android:lint"],
|
|
**lintargs,
|
|
)
|
|
|
|
|
|
def report_gradlew(config, fix, subdir, lint_tasks=[], **lintargs):
|
|
topsrcdir = lintargs["root"]
|
|
topobjdir = lintargs["topobjdir"]
|
|
|
|
if fix:
|
|
tasks = ["ktlintFormat", "detekt"]
|
|
else:
|
|
tasks = ["ktlint", "detekt"]
|
|
|
|
extra_args = lintargs.get("extra_args") or []
|
|
|
|
gradle(
|
|
lintargs["log"],
|
|
topsrcdir=topsrcdir,
|
|
topobjdir=topobjdir,
|
|
tasks=tasks,
|
|
extra_args=extra_args + ["-p", os.path.join(topsrcdir, subdir), "--continue"],
|
|
)
|
|
|
|
reports = os.path.join(topsrcdir, subdir, "build", "reports")
|
|
results = []
|
|
|
|
excludes = []
|
|
for path in EXCLUSION_FILES:
|
|
with open(os.path.join(topsrcdir, path)) as fh:
|
|
for f in fh.readlines():
|
|
if "*" in f:
|
|
excludes.extend(glob.glob(f.strip()))
|
|
elif f.startswith(subdir):
|
|
excludes.append(f.strip())
|
|
|
|
detekt_report = None
|
|
if os.path.exists(os.path.join(reports, "detekt", "detekt.xml")):
|
|
detekt_report = os.path.join(reports, "detekt", "detekt.xml")
|
|
elif os.path.join(
|
|
topobjdir, "gradle", "build", subdir, "reports", "detekt", "detekt.xml"
|
|
):
|
|
detekt_report = os.path.join(
|
|
topobjdir, "gradle", "build", subdir, "reports", "detekt", "detekt.xml"
|
|
)
|
|
else:
|
|
print(f"Could not read detekt report: '{detekt_report}'")
|
|
pass
|
|
|
|
try:
|
|
tree = ET.parse(open(detekt_report))
|
|
root = tree.getroot()
|
|
|
|
for file in root.findall("file"):
|
|
name = file.get("name")
|
|
if is_excluded_file(topsrcdir, excludes, name):
|
|
continue
|
|
for error in file:
|
|
err = {
|
|
"rule": error.get("source"),
|
|
"path": name,
|
|
"lineno": int(error.get("line") or 0),
|
|
"column": int(error.get("column") or 0),
|
|
"message": error.get("message"),
|
|
"level": "error",
|
|
}
|
|
results.append(result.from_config(config, **err))
|
|
except FileNotFoundError:
|
|
print(f"Could not read detekt report: '{detekt_report}'")
|
|
pass
|
|
|
|
ktlint_file = "ktlint.json"
|
|
if fix:
|
|
ktlint_file = "ktlintFormat.json"
|
|
try:
|
|
issues = json.load(
|
|
open(
|
|
os.path.join(
|
|
reports,
|
|
"ktlint",
|
|
ktlint_file,
|
|
),
|
|
)
|
|
)
|
|
|
|
for issue in issues:
|
|
name = issue["file"]
|
|
if is_excluded_file(topsrcdir, excludes, name):
|
|
continue
|
|
for error in issue["errors"]:
|
|
err = {
|
|
"rule": error["rule"],
|
|
"path": name,
|
|
"lineno": error["line"],
|
|
"column": error["column"],
|
|
"message": error["message"],
|
|
"level": "error",
|
|
}
|
|
results.append(result.from_config(config, **err))
|
|
except FileNotFoundError:
|
|
print(f"Could not read ktlint report: `{ktlint_file}`")
|
|
pass
|
|
|
|
return results + read_lint_report(config, subdir, tasks=lint_tasks, **lintargs)
|
|
|
|
|
|
def is_excluded_file(topsrcdir, excludes, file):
|
|
for path in excludes:
|
|
if file.startswith(os.path.join(topsrcdir, path)):
|
|
return True
|
|
return False
|
|
|
|
|
|
def api_lint(_paths, config, **lintargs):
|
|
topsrcdir = lintargs["root"]
|
|
topobjdir = lintargs["topobjdir"]
|
|
|
|
gradle(
|
|
lintargs["log"],
|
|
topsrcdir=topsrcdir,
|
|
topobjdir=topobjdir,
|
|
tasks=lintargs["substs"]["GRADLE_ANDROID_API_LINT_TASKS"],
|
|
extra_args=lintargs.get("extra_args") or [],
|
|
)
|
|
|
|
folder = lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_APILINT_FOLDER"]
|
|
|
|
results = []
|
|
|
|
with open(os.path.join(topobjdir, folder, "apilint-result.json")) as f:
|
|
issues = json.load(f)
|
|
|
|
for rule in ("compat_failures", "failures"):
|
|
for r in issues[rule]:
|
|
err = {
|
|
"rule": r["rule"] if rule == "failures" else "compat_failures",
|
|
"path": r["file"],
|
|
"lineno": int(r["line"]),
|
|
"column": int(r.get("column") or 0),
|
|
"message": r["msg"],
|
|
"level": "error" if r["error"] else "warning",
|
|
}
|
|
results.append(result.from_config(config, **err))
|
|
|
|
for r in issues["api_changes"]:
|
|
err = {
|
|
"rule": "api_changes",
|
|
"path": r["file"],
|
|
"lineno": int(r["line"]),
|
|
"column": int(r.get("column") or 0),
|
|
"message": "Unexpected api change. Please run ./mach gradle {} for more "
|
|
"information".format(
|
|
" ".join(lintargs["substs"]["GRADLE_ANDROID_API_LINT_TASKS"])
|
|
),
|
|
}
|
|
results.append(result.from_config(config, **err))
|
|
|
|
return results
|
|
|
|
|
|
def javadoc(_paths, config, **lintargs):
|
|
topsrcdir = lintargs["root"]
|
|
topobjdir = lintargs["topobjdir"]
|
|
|
|
gradle(
|
|
lintargs["log"],
|
|
topsrcdir=topsrcdir,
|
|
topobjdir=topobjdir,
|
|
tasks=lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_DOCS_TASKS"],
|
|
extra_args=lintargs.get("extra_args") or [],
|
|
)
|
|
|
|
output_files = lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_DOCS_OUTPUT_FILES"]
|
|
|
|
results = []
|
|
|
|
for output_file in output_files:
|
|
with open(os.path.join(topobjdir, output_file)) as f:
|
|
# Like: '[{"path":"/absolute/path/to/topsrcdir/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/ContentBlocking.java","lineno":"462","level":"warning","message":"no @return"}]'. # NOQA: E501
|
|
issues = json.load(f)
|
|
|
|
for issue in issues:
|
|
# We want warnings to be errors for linting purposes.
|
|
# TODO: Bug 1316188 - resolve missing javadoc comments
|
|
issue["level"] = (
|
|
"error" if issue["message"] != ": no comment" else "warning"
|
|
)
|
|
results.append(result.from_config(config, **issue))
|
|
|
|
return results
|
|
|
|
|
|
def lint(_paths, config, **lintargs):
|
|
topsrcdir = lintargs["root"]
|
|
topobjdir = lintargs["topobjdir"]
|
|
|
|
gradle(
|
|
lintargs["log"],
|
|
topsrcdir=topsrcdir,
|
|
topobjdir=topobjdir,
|
|
tasks=lintargs["substs"]["GRADLE_ANDROID_LINT_TASKS"],
|
|
extra_args=lintargs.get("extra_args") or [],
|
|
)
|
|
|
|
path = os.path.join(
|
|
lintargs["topobjdir"],
|
|
"gradle/build/mobile/android/geckoview/reports",
|
|
"lint-results-{}.xml".format(
|
|
lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_VARIANT_NAME"]
|
|
),
|
|
)
|
|
tree = ET.parse(open(path))
|
|
root = tree.getroot()
|
|
|
|
results = []
|
|
|
|
for issue in root.findall("issue"):
|
|
location = issue[0]
|
|
if "third_party" in location.get("file") or "thirdparty" in location.get(
|
|
"file"
|
|
):
|
|
continue
|
|
err = {
|
|
"level": issue.get("severity").lower(),
|
|
"rule": issue.get("id"),
|
|
"message": issue.get("message"),
|
|
"path": location.get("file"),
|
|
"lineno": int(location.get("line") or 0),
|
|
}
|
|
results.append(result.from_config(config, **err))
|
|
|
|
return results
|
|
|
|
|
|
def read_lint_report(config, subdir, tasks=[], **lintargs):
|
|
topsrcdir = lintargs["root"]
|
|
topobjdir = lintargs["topobjdir"]
|
|
|
|
gradle(
|
|
lintargs["log"],
|
|
topsrcdir=topsrcdir,
|
|
topobjdir=topobjdir,
|
|
tasks=tasks,
|
|
extra_args=lintargs.get("extra_args") or [],
|
|
)
|
|
|
|
reports = os.path.join(topsrcdir, subdir, "build", "reports")
|
|
|
|
excludes = []
|
|
for path in EXCLUSION_FILES:
|
|
with open(os.path.join(topsrcdir, path)) as fh:
|
|
for f in fh.readlines():
|
|
if "*" in f:
|
|
excludes.extend(glob.glob(f.strip()))
|
|
elif f.startswith(subdir):
|
|
excludes.append(f.strip())
|
|
|
|
try:
|
|
files = os.listdir(
|
|
os.path.join(
|
|
reports,
|
|
"lint",
|
|
)
|
|
)
|
|
|
|
results = []
|
|
for file in files:
|
|
data = json.load(
|
|
open(
|
|
os.path.join(reports, "lint", file),
|
|
)
|
|
).get(
|
|
"runs", [{}]
|
|
)[0]
|
|
|
|
issues = data.get("results", [])
|
|
rules = data.get("tool", {}).get("driver", {}).get("rules", [])
|
|
|
|
for issue in issues:
|
|
dir = os.path.join(topsrcdir, subdir)
|
|
if subdir != os.path.join("mobile", "android", "android-components"):
|
|
dir = os.path.join(topsrcdir, "mobile", "android")
|
|
name = os.path.join(
|
|
dir,
|
|
issue.get("locations", [{}])[0]
|
|
.get("physicalLocation", {})
|
|
.get("artifactLocation", {})
|
|
.get("uri"),
|
|
)
|
|
|
|
if is_excluded_file(topsrcdir, excludes, name) and not "/res/" in name:
|
|
continue
|
|
|
|
level = "error"
|
|
if "level" in issue:
|
|
level = issue["level"]
|
|
elif "ruleIndex" in issue and len(rules) > issue["ruleIndex"]:
|
|
rule_level = (
|
|
rules[issue["ruleIndex"]]
|
|
.get("defaultConfiguration", {})
|
|
.get("level")
|
|
)
|
|
if rule_level:
|
|
level = rule_level
|
|
|
|
err = {
|
|
"rule": issue.get("ruleId"),
|
|
"path": name,
|
|
"lineno": issue.get("locations", [{}])[0]
|
|
.get("physicalLocation", {})
|
|
.get("region", {})
|
|
.get("startLine"),
|
|
"column": issue.get("locations", [{}])[0]
|
|
.get("physicalLocation", {})
|
|
.get("region", {})
|
|
.get("startColumn"),
|
|
"message": issue.get("message", {}).get("text"),
|
|
"level": level,
|
|
}
|
|
results.append(result.from_config(config, **err))
|
|
return results
|
|
except FileNotFoundError:
|
|
print("Could not read lint report from ", subdir)
|
|
return []
|
|
|
|
|
|
def _parse_checkstyle_output(config, topsrcdir=None, report_path=None):
|
|
tree = ET.parse(open(report_path))
|
|
root = tree.getroot()
|
|
|
|
for file in root.findall("file"):
|
|
for error in file.findall("error"):
|
|
# Like <error column="42" line="22" message="Name 'mPorts' must match pattern 'xm[A-Z][A-Za-z]*$'." severity="error" source="com.puppycrawl.tools.checkstyle.checks.naming.MemberNameCheck" />. # NOQA: E501
|
|
err = {
|
|
"level": "error",
|
|
"rule": error.get("source"),
|
|
"message": error.get("message"),
|
|
"path": file.get("name"),
|
|
"lineno": int(error.get("line") or 0),
|
|
"column": int(error.get("column") or 0),
|
|
}
|
|
yield result.from_config(config, **err)
|
|
|
|
|
|
def checkstyle(_paths, config, **lintargs):
|
|
topsrcdir = lintargs["root"]
|
|
topobjdir = lintargs["topobjdir"]
|
|
|
|
gradle(
|
|
lintargs["log"],
|
|
topsrcdir=topsrcdir,
|
|
topobjdir=topobjdir,
|
|
tasks=lintargs["substs"]["GRADLE_ANDROID_CHECKSTYLE_TASKS"],
|
|
extra_args=lintargs.get("extra_args") or [],
|
|
)
|
|
|
|
results = []
|
|
|
|
for relative_path in lintargs["substs"]["GRADLE_ANDROID_CHECKSTYLE_OUTPUT_FILES"]:
|
|
report_path = os.path.join(lintargs["topobjdir"], relative_path)
|
|
results.extend(
|
|
_parse_checkstyle_output(
|
|
config, topsrcdir=lintargs["root"], report_path=report_path
|
|
)
|
|
)
|
|
|
|
return results
|
|
|
|
|
|
def _parse_android_test_results(config, topsrcdir=None, report_dir=None):
|
|
# A brute force way to turn a Java FQN into a path on disk. Assumes Java
|
|
# and Kotlin sources are in mobile/android for performance and simplicity.
|
|
sourcepath_finder = FileFinder(os.path.join(topsrcdir, "mobile", "android"))
|
|
|
|
finder = FileFinder(report_dir)
|
|
reports = list(finder.find("TEST-*.xml"))
|
|
if not reports:
|
|
raise RuntimeError(f"No reports found under {report_dir}")
|
|
|
|
for report, _ in reports:
|
|
tree = ET.parse(open(os.path.join(finder.base, report)))
|
|
root = tree.getroot()
|
|
|
|
class_name = root.get(
|
|
"name"
|
|
) # Like 'org.mozilla.gecko.permissions.TestPermissions'.
|
|
path = (
|
|
"**/" + class_name.replace(".", "/") + ".*"
|
|
) # Like '**/org/mozilla/gecko/permissions/TestPermissions.*'. # NOQA: E501
|
|
|
|
for testcase in root.findall("testcase"):
|
|
function_name = testcase.get("name")
|
|
|
|
# Schema cribbed from http://llg.cubic.org/docs/junit/.
|
|
for unexpected in itertools.chain(
|
|
testcase.findall("error"), testcase.findall("failure")
|
|
):
|
|
sourcepaths = list(sourcepath_finder.find(path))
|
|
if not sourcepaths:
|
|
raise RuntimeError(f"No sourcepath found for class {class_name}")
|
|
|
|
for sourcepath, _ in sourcepaths:
|
|
lineno = 0
|
|
message = unexpected.get("message")
|
|
# Turn '... at org.mozilla.gecko.permissions.TestPermissions.testMultipleRequestsAreQueuedAndDispatchedSequentially(TestPermissions.java:118)' into 118. # NOQA: E501
|
|
pattern = r"at {class_name}\.{function_name}\(.*:(\d+)\)"
|
|
pattern = pattern.format(
|
|
class_name=class_name, function_name=function_name
|
|
)
|
|
match = re.search(pattern, message)
|
|
if match:
|
|
lineno = int(match.group(1))
|
|
else:
|
|
msg = f"No source line found for {class_name}.{function_name}"
|
|
raise RuntimeError(msg)
|
|
|
|
err = {
|
|
"level": "error",
|
|
"rule": unexpected.get("type"),
|
|
"message": message,
|
|
"path": os.path.join(
|
|
topsrcdir, "mobile", "android", sourcepath
|
|
),
|
|
"lineno": lineno,
|
|
}
|
|
yield result.from_config(config, **err)
|
|
|
|
|
|
def test(_paths, config, **lintargs):
|
|
topsrcdir = lintargs["root"]
|
|
topobjdir = lintargs["topobjdir"]
|
|
|
|
gradle(
|
|
lintargs["log"],
|
|
topsrcdir=topsrcdir,
|
|
topobjdir=topobjdir,
|
|
tasks=lintargs["substs"]["GRADLE_ANDROID_TEST_TASKS"],
|
|
extra_args=lintargs.get("extra_args") or [],
|
|
)
|
|
|
|
results = []
|
|
|
|
def capitalize(s):
|
|
# Can't use str.capitalize because it lower cases trailing letters.
|
|
return (s[0].upper() + s[1:]) if s else ""
|
|
|
|
pairs = [("geckoview", lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_VARIANT_NAME"])]
|
|
for project, variant in pairs:
|
|
report_dir = os.path.join(
|
|
lintargs["topobjdir"],
|
|
f"gradle/build/mobile/android/{project}/test-results/test{capitalize(variant)}UnitTest",
|
|
)
|
|
results.extend(
|
|
_parse_android_test_results(
|
|
config, topsrcdir=lintargs["root"], report_dir=report_dir
|
|
)
|
|
)
|
|
|
|
return results
|