diff options
Diffstat (limited to '')
| -rw-r--r-- | tools/lint/android/lints.py | 421 |
1 files changed, 421 insertions, 0 deletions
diff --git a/tools/lint/android/lints.py b/tools/lint/android/lints.py new file mode 100644 index 0000000000..a0ec59e805 --- /dev/null +++ b/tools/lint/android/lints.py @@ -0,0 +1,421 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import glob +import itertools +import json +import os +import re +import subprocess +import sys +import xml.etree.ElementTree as ET + +import mozpack.path as mozpath +import six +from mozlint import result +from mozpack.files import FileFinder + +# The Gradle target invocations are serialized with a simple locking file scheme. It's fine for +# them to take a while, since the first will compile all the Java, etc, and then perform +# potentially expensive static analyses. +GRADLE_LOCK_MAX_WAIT_SECONDS = 20 * 60 + + +def setup(root, **setupargs): + if setupargs.get("substs", {}).get("MOZ_BUILD_APP") != "mobile/android": + return 1 + + if "topobjdir" not in setupargs: + print( + "Skipping {}: a configured Android build is required!".format( + setupargs["name"] + ) + ) + return 1 + + return 0 + + +def gradle(log, topsrcdir=None, topobjdir=None, tasks=[], extra_args=[], verbose=True): + sys.path.insert(0, os.path.join(topsrcdir, "mobile", "android")) + from gradle import gradle_lock + + with gradle_lock(topobjdir, max_wait_seconds=GRADLE_LOCK_MAX_WAIT_SECONDS): + # The android-lint parameter can be used by gradle tasks to run special + # logic when they are run for a lint using + # project.hasProperty('android-lint') + cmd_args = ( + [ + sys.executable, + os.path.join(topsrcdir, "mach"), + "gradle", + "--verbose", + "-Pandroid-lint", + "--", + ] + + tasks + + extra_args + ) + + cmd = " ".join(six.moves.shlex_quote(arg) for arg in cmd_args) + log.debug(cmd) + + # Gradle and mozprocess do not get along well, so we use subprocess + # directly. + proc = subprocess.Popen(cmd_args, cwd=topsrcdir) + status = None + # Leave it to the subprocess to handle Ctrl+C. If it terminates as a result + # of Ctrl+C, proc.wait() will return a status code, and, we get out of the + # loop. If it doesn't, like e.g. gdb, we continue waiting. + while status is None: + try: + status = proc.wait() + except KeyboardInterrupt: + pass + + try: + proc.wait() + except KeyboardInterrupt: + proc.kill() + raise + + return proc.returncode + + +def format(config, fix=None, **lintargs): + topsrcdir = lintargs["root"] + topobjdir = lintargs["topobjdir"] + + if fix: + tasks = lintargs["substs"]["GRADLE_ANDROID_FORMAT_LINT_FIX_TASKS"] + else: + tasks = lintargs["substs"]["GRADLE_ANDROID_FORMAT_LINT_CHECK_TASKS"] + + ret = gradle( + lintargs["log"], + topsrcdir=topsrcdir, + topobjdir=topobjdir, + tasks=tasks, + extra_args=lintargs.get("extra_args") or [], + ) + + results = [] + for path in lintargs["substs"]["GRADLE_ANDROID_FORMAT_LINT_FOLDERS"]: + folder = os.path.join( + topobjdir, "gradle", "build", path, "spotless", "spotlessJava" + ) + for filename in glob.iglob(folder + "/**/*.java", recursive=True): + err = { + "rule": "spotless-java", + "path": os.path.join( + topsrcdir, path, mozpath.relpath(filename, folder) + ), + "lineno": 0, + "column": 0, + "message": "Formatting error, please run ./mach lint -l android-format --fix", + "level": "error", + } + results.append(result.from_config(config, **err)) + folder = os.path.join( + topobjdir, "gradle", "build", path, "spotless", "spotlessKotlin" + ) + for filename in glob.iglob(folder + "/**/*.kt", recursive=True): + err = { + "rule": "spotless-kt", + "path": os.path.join( + topsrcdir, path, mozpath.relpath(filename, folder) + ), + "lineno": 0, + "column": 0, + "message": "Formatting error, please run ./mach lint -l android-format --fix", + "level": "error", + } + results.append(result.from_config(config, **err)) + + if len(results) == 0 and ret != 0: + # spotless seems to hit unfixed error. + err = { + "rule": "spotless", + "path": "", + "lineno": 0, + "column": 0, + "message": "Unexpected error", + "level": "error", + } + results.append(result.from_config(config, **err)) + + # If --fix was passed, we just report the number of files that were changed + if fix: + return {"results": [], "fixed": len(results)} + return results + + +def api_lint(config, **lintargs): + topsrcdir = lintargs["root"] + topobjdir = lintargs["topobjdir"] + + gradle( + lintargs["log"], + topsrcdir=topsrcdir, + topobjdir=topobjdir, + tasks=lintargs["substs"]["GRADLE_ANDROID_API_LINT_TASKS"], + extra_args=lintargs.get("extra_args") or [], + ) + + folder = lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_APILINT_FOLDER"] + + results = [] + + with open(os.path.join(topobjdir, folder, "apilint-result.json")) as f: + issues = json.load(f) + + for rule in ("compat_failures", "failures"): + for r in issues[rule]: + err = { + "rule": r["rule"] if rule == "failures" else "compat_failures", + "path": r["file"], + "lineno": int(r["line"]), + "column": int(r.get("column") or 0), + "message": r["msg"], + "level": "error" if r["error"] else "warning", + } + results.append(result.from_config(config, **err)) + + for r in issues["api_changes"]: + err = { + "rule": "api_changes", + "path": r["file"], + "lineno": int(r["line"]), + "column": int(r.get("column") or 0), + "message": "Unexpected api change. Please run ./mach gradle {} for more " + "information".format( + " ".join(lintargs["substs"]["GRADLE_ANDROID_API_LINT_TASKS"]) + ), + } + results.append(result.from_config(config, **err)) + + return results + + +def javadoc(config, **lintargs): + topsrcdir = lintargs["root"] + topobjdir = lintargs["topobjdir"] + + gradle( + lintargs["log"], + topsrcdir=topsrcdir, + topobjdir=topobjdir, + tasks=lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_DOCS_TASKS"], + extra_args=lintargs.get("extra_args") or [], + ) + + output_files = lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_DOCS_OUTPUT_FILES"] + + results = [] + + for output_file in output_files: + with open(os.path.join(topobjdir, output_file)) as f: + # Like: '[{"path":"/absolute/path/to/topsrcdir/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/ContentBlocking.java","lineno":"462","level":"warning","message":"no @return"}]'. # NOQA: E501 + issues = json.load(f) + + for issue in issues: + # We want warnings to be errors for linting purposes. + # TODO: Bug 1316188 - resolve missing javadoc comments + issue["level"] = ( + "error" if issue["message"] != ": no comment" else "warning" + ) + results.append(result.from_config(config, **issue)) + + return results + + +def lint(config, **lintargs): + topsrcdir = lintargs["root"] + topobjdir = lintargs["topobjdir"] + + gradle( + lintargs["log"], + topsrcdir=topsrcdir, + topobjdir=topobjdir, + tasks=lintargs["substs"]["GRADLE_ANDROID_LINT_TASKS"], + extra_args=lintargs.get("extra_args") or [], + ) + + # It's surprising that this is the App variant name, but this is "withoutGeckoBinariesDebug" + # right now and the GeckoView variant name is "withGeckoBinariesDebug". This will be addressed + # as we unify variants. + path = os.path.join( + lintargs["topobjdir"], + "gradle/build/mobile/android/geckoview/reports", + "lint-results-{}.xml".format( + lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_VARIANT_NAME"] + ), + ) + tree = ET.parse(open(path, "rt")) + root = tree.getroot() + + results = [] + + for issue in root.findall("issue"): + location = issue[0] + if "third_party" in location.get("file") or "thirdparty" in location.get( + "file" + ): + continue + err = { + "level": issue.get("severity").lower(), + "rule": issue.get("id"), + "message": issue.get("message"), + "path": location.get("file"), + "lineno": int(location.get("line") or 0), + } + results.append(result.from_config(config, **err)) + + return results + + +def _parse_checkstyle_output(config, topsrcdir=None, report_path=None): + tree = ET.parse(open(report_path, "rt")) + root = tree.getroot() + + for file in root.findall("file"): + + for error in file.findall("error"): + # Like <error column="42" line="22" message="Name 'mPorts' must match pattern 'xm[A-Z][A-Za-z]*$'." severity="error" source="com.puppycrawl.tools.checkstyle.checks.naming.MemberNameCheck" />. # NOQA: E501 + err = { + "level": "error", + "rule": error.get("source"), + "message": error.get("message"), + "path": file.get("name"), + "lineno": int(error.get("line") or 0), + "column": int(error.get("column") or 0), + } + yield result.from_config(config, **err) + + +def checkstyle(config, **lintargs): + topsrcdir = lintargs["root"] + topobjdir = lintargs["topobjdir"] + + gradle( + lintargs["log"], + topsrcdir=topsrcdir, + topobjdir=topobjdir, + tasks=lintargs["substs"]["GRADLE_ANDROID_CHECKSTYLE_TASKS"], + extra_args=lintargs.get("extra_args") or [], + ) + + results = [] + + for relative_path in lintargs["substs"]["GRADLE_ANDROID_CHECKSTYLE_OUTPUT_FILES"]: + report_path = os.path.join(lintargs["topobjdir"], relative_path) + results.extend( + _parse_checkstyle_output( + config, topsrcdir=lintargs["root"], report_path=report_path + ) + ) + + return results + + +def _parse_android_test_results(config, topsrcdir=None, report_dir=None): + # A brute force way to turn a Java FQN into a path on disk. Assumes Java + # and Kotlin sources are in mobile/android for performance and simplicity. + sourcepath_finder = FileFinder(os.path.join(topsrcdir, "mobile", "android")) + + finder = FileFinder(report_dir) + reports = list(finder.find("TEST-*.xml")) + if not reports: + raise RuntimeError("No reports found under {}".format(report_dir)) + + for report, _ in reports: + tree = ET.parse(open(os.path.join(finder.base, report), "rt")) + root = tree.getroot() + + class_name = root.get( + "name" + ) # Like 'org.mozilla.gecko.permissions.TestPermissions'. + path = ( + "**/" + class_name.replace(".", "/") + ".*" + ) # Like '**/org/mozilla/gecko/permissions/TestPermissions.*'. # NOQA: E501 + + for testcase in root.findall("testcase"): + function_name = testcase.get("name") + + # Schema cribbed from http://llg.cubic.org/docs/junit/. + for unexpected in itertools.chain( + testcase.findall("error"), testcase.findall("failure") + ): + sourcepaths = list(sourcepath_finder.find(path)) + if not sourcepaths: + raise RuntimeError( + "No sourcepath found for class {class_name}".format( + class_name=class_name + ) + ) + + for sourcepath, _ in sourcepaths: + lineno = 0 + message = unexpected.get("message") + # Turn '... at org.mozilla.gecko.permissions.TestPermissions.testMultipleRequestsAreQueuedAndDispatchedSequentially(TestPermissions.java:118)' into 118. # NOQA: E501 + pattern = r"at {class_name}\.{function_name}\(.*:(\d+)\)" + pattern = pattern.format( + class_name=class_name, function_name=function_name + ) + match = re.search(pattern, message) + if match: + lineno = int(match.group(1)) + else: + msg = "No source line found for {class_name}.{function_name}".format( + class_name=class_name, function_name=function_name + ) + raise RuntimeError(msg) + + err = { + "level": "error", + "rule": unexpected.get("type"), + "message": message, + "path": os.path.join( + topsrcdir, "mobile", "android", sourcepath + ), + "lineno": lineno, + } + yield result.from_config(config, **err) + + +def test(config, **lintargs): + topsrcdir = lintargs["root"] + topobjdir = lintargs["topobjdir"] + + gradle( + lintargs["log"], + topsrcdir=topsrcdir, + topobjdir=topobjdir, + tasks=lintargs["substs"]["GRADLE_ANDROID_TEST_TASKS"], + extra_args=lintargs.get("extra_args") or [], + ) + + results = [] + + def capitalize(s): + # Can't use str.capitalize because it lower cases trailing letters. + return (s[0].upper() + s[1:]) if s else "" + + pairs = [("geckoview", lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_VARIANT_NAME"])] + for project, variant in pairs: + report_dir = os.path.join( + lintargs["topobjdir"], + "gradle/build/mobile/android/{}/test-results/test{}UnitTest".format( + project, capitalize(variant) + ), + ) + results.extend( + _parse_android_test_results( + config, topsrcdir=lintargs["root"], report_dir=report_dir + ) + ) + + return results |