Adding upstream version 115.7.0esr.upstream/115.7.0esr

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

2 files changed, 421 insertions, 0 deletions

diff --git a/tools/lint/android/__init__.py b/tools/lint/android/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/tools/lint/android/__init__.py
diff --git a/tools/lint/android/lints.py b/tools/lint/android/lints.py
new file mode 100644
index 0000000000..a0ec59e805
--- /dev/null
+++ b/tools/lint/android/lints.py
@@ -0,0 +1,421 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import glob
+import itertools
+import json
+import os
+import re
+import subprocess
+import sys
+import xml.etree.ElementTree as ET
+
+import mozpack.path as mozpath
+import six
+from mozlint import result
+from mozpack.files import FileFinder
+
+# The Gradle target invocations are serialized with a simple locking file scheme.  It's fine for
+# them to take a while, since the first will compile all the Java, etc, and then perform
+# potentially expensive static analyses.
+GRADLE_LOCK_MAX_WAIT_SECONDS = 20 * 60
+
+
+def setup(root, **setupargs):
+    if setupargs.get("substs", {}).get("MOZ_BUILD_APP") != "mobile/android":
+        return 1
+
+    if "topobjdir" not in setupargs:
+        print(
+            "Skipping {}: a configured Android build is required!".format(
+                setupargs["name"]
+            )
+        )
+        return 1
+
+    return 0
+
+
+def gradle(log, topsrcdir=None, topobjdir=None, tasks=[], extra_args=[], verbose=True):
+    sys.path.insert(0, os.path.join(topsrcdir, "mobile", "android"))
+    from gradle import gradle_lock
+
+    with gradle_lock(topobjdir, max_wait_seconds=GRADLE_LOCK_MAX_WAIT_SECONDS):
+        # The android-lint parameter can be used by gradle tasks to run special
+        # logic when they are run for a lint using
+        #   project.hasProperty('android-lint')
+        cmd_args = (
+            [
+                sys.executable,
+                os.path.join(topsrcdir, "mach"),
+                "gradle",
+                "--verbose",
+                "-Pandroid-lint",
+                "--",
+            ]
+            + tasks
+            + extra_args
+        )
+
+        cmd = " ".join(six.moves.shlex_quote(arg) for arg in cmd_args)
+        log.debug(cmd)
+
+        # Gradle and mozprocess do not get along well, so we use subprocess
+        # directly.
+        proc = subprocess.Popen(cmd_args, cwd=topsrcdir)
+        status = None
+        # Leave it to the subprocess to handle Ctrl+C. If it terminates as a result
+        # of Ctrl+C, proc.wait() will return a status code, and, we get out of the
+        # loop. If it doesn't, like e.g. gdb, we continue waiting.
+        while status is None:
+            try:
+                status = proc.wait()
+            except KeyboardInterrupt:
+                pass
+
+        try:
+            proc.wait()
+        except KeyboardInterrupt:
+            proc.kill()
+            raise
+
+        return proc.returncode
+
+
+def format(config, fix=None, **lintargs):
+    topsrcdir = lintargs["root"]
+    topobjdir = lintargs["topobjdir"]
+
+    if fix:
+        tasks = lintargs["substs"]["GRADLE_ANDROID_FORMAT_LINT_FIX_TASKS"]
+    else:
+        tasks = lintargs["substs"]["GRADLE_ANDROID_FORMAT_LINT_CHECK_TASKS"]
+
+    ret = gradle(
+        lintargs["log"],
+        topsrcdir=topsrcdir,
+        topobjdir=topobjdir,
+        tasks=tasks,
+        extra_args=lintargs.get("extra_args") or [],
+    )
+
+    results = []
+    for path in lintargs["substs"]["GRADLE_ANDROID_FORMAT_LINT_FOLDERS"]:
+        folder = os.path.join(
+            topobjdir, "gradle", "build", path, "spotless", "spotlessJava"
+        )
+        for filename in glob.iglob(folder + "/**/*.java", recursive=True):
+            err = {
+                "rule": "spotless-java",
+                "path": os.path.join(
+                    topsrcdir, path, mozpath.relpath(filename, folder)
+                ),
+                "lineno": 0,
+                "column": 0,
+                "message": "Formatting error, please run ./mach lint -l android-format --fix",
+                "level": "error",
+            }
+            results.append(result.from_config(config, **err))
+        folder = os.path.join(
+            topobjdir, "gradle", "build", path, "spotless", "spotlessKotlin"
+        )
+        for filename in glob.iglob(folder + "/**/*.kt", recursive=True):
+            err = {
+                "rule": "spotless-kt",
+                "path": os.path.join(
+                    topsrcdir, path, mozpath.relpath(filename, folder)
+                ),
+                "lineno": 0,
+                "column": 0,
+                "message": "Formatting error, please run ./mach lint -l android-format --fix",
+                "level": "error",
+            }
+            results.append(result.from_config(config, **err))
+
+    if len(results) == 0 and ret != 0:
+        # spotless seems to hit unfixed error.
+        err = {
+            "rule": "spotless",
+            "path": "",
+            "lineno": 0,
+            "column": 0,
+            "message": "Unexpected error",
+            "level": "error",
+        }
+        results.append(result.from_config(config, **err))
+
+    # If --fix was passed, we just report the number of files that were changed
+    if fix:
+        return {"results": [], "fixed": len(results)}
+    return results
+
+
+def api_lint(config, **lintargs):
+    topsrcdir = lintargs["root"]
+    topobjdir = lintargs["topobjdir"]
+
+    gradle(
+        lintargs["log"],
+        topsrcdir=topsrcdir,
+        topobjdir=topobjdir,
+        tasks=lintargs["substs"]["GRADLE_ANDROID_API_LINT_TASKS"],
+        extra_args=lintargs.get("extra_args") or [],
+    )
+
+    folder = lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_APILINT_FOLDER"]
+
+    results = []
+
+    with open(os.path.join(topobjdir, folder, "apilint-result.json")) as f:
+        issues = json.load(f)
+
+        for rule in ("compat_failures", "failures"):
+            for r in issues[rule]:
+                err = {
+                    "rule": r["rule"] if rule == "failures" else "compat_failures",
+                    "path": r["file"],
+                    "lineno": int(r["line"]),
+                    "column": int(r.get("column") or 0),
+                    "message": r["msg"],
+                    "level": "error" if r["error"] else "warning",
+                }
+                results.append(result.from_config(config, **err))
+
+        for r in issues["api_changes"]:
+            err = {
+                "rule": "api_changes",
+                "path": r["file"],
+                "lineno": int(r["line"]),
+                "column": int(r.get("column") or 0),
+                "message": "Unexpected api change. Please run ./mach gradle {} for more "
+                "information".format(
+                    " ".join(lintargs["substs"]["GRADLE_ANDROID_API_LINT_TASKS"])
+                ),
+            }
+            results.append(result.from_config(config, **err))
+
+    return results
+
+
+def javadoc(config, **lintargs):
+    topsrcdir = lintargs["root"]
+    topobjdir = lintargs["topobjdir"]
+
+    gradle(
+        lintargs["log"],
+        topsrcdir=topsrcdir,
+        topobjdir=topobjdir,
+        tasks=lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_DOCS_TASKS"],
+        extra_args=lintargs.get("extra_args") or [],
+    )
+
+    output_files = lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_DOCS_OUTPUT_FILES"]
+
+    results = []
+
+    for output_file in output_files:
+        with open(os.path.join(topobjdir, output_file)) as f:
+            # Like: '[{"path":"/absolute/path/to/topsrcdir/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/ContentBlocking.java","lineno":"462","level":"warning","message":"no @return"}]'.  # NOQA: E501
+            issues = json.load(f)
+
+            for issue in issues:
+                # We want warnings to be errors for linting purposes.
+                # TODO: Bug 1316188 - resolve missing javadoc comments
+                issue["level"] = (
+                    "error" if issue["message"] != ": no comment" else "warning"
+                )
+                results.append(result.from_config(config, **issue))
+
+    return results
+
+
+def lint(config, **lintargs):
+    topsrcdir = lintargs["root"]
+    topobjdir = lintargs["topobjdir"]
+
+    gradle(
+        lintargs["log"],
+        topsrcdir=topsrcdir,
+        topobjdir=topobjdir,
+        tasks=lintargs["substs"]["GRADLE_ANDROID_LINT_TASKS"],
+        extra_args=lintargs.get("extra_args") or [],
+    )
+
+    # It's surprising that this is the App variant name, but this is "withoutGeckoBinariesDebug"
+    # right now and the GeckoView variant name is "withGeckoBinariesDebug".  This will be addressed
+    # as we unify variants.
+    path = os.path.join(
+        lintargs["topobjdir"],
+        "gradle/build/mobile/android/geckoview/reports",
+        "lint-results-{}.xml".format(
+            lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_VARIANT_NAME"]
+        ),
+    )
+    tree = ET.parse(open(path, "rt"))
+    root = tree.getroot()
+
+    results = []
+
+    for issue in root.findall("issue"):
+        location = issue[0]
+        if "third_party" in location.get("file") or "thirdparty" in location.get(
+            "file"
+        ):
+            continue
+        err = {
+            "level": issue.get("severity").lower(),
+            "rule": issue.get("id"),
+            "message": issue.get("message"),
+            "path": location.get("file"),
+            "lineno": int(location.get("line") or 0),
+        }
+        results.append(result.from_config(config, **err))
+
+    return results
+
+
+def _parse_checkstyle_output(config, topsrcdir=None, report_path=None):
+    tree = ET.parse(open(report_path, "rt"))
+    root = tree.getroot()
+
+    for file in root.findall("file"):
+
+        for error in file.findall("error"):
+            # Like <error column="42" line="22" message="Name 'mPorts' must match pattern 'xm[A-Z][A-Za-z]*$'." severity="error" source="com.puppycrawl.tools.checkstyle.checks.naming.MemberNameCheck" />.  # NOQA: E501
+            err = {
+                "level": "error",
+                "rule": error.get("source"),
+                "message": error.get("message"),
+                "path": file.get("name"),
+                "lineno": int(error.get("line") or 0),
+                "column": int(error.get("column") or 0),
+            }
+            yield result.from_config(config, **err)
+
+
+def checkstyle(config, **lintargs):
+    topsrcdir = lintargs["root"]
+    topobjdir = lintargs["topobjdir"]
+
+    gradle(
+        lintargs["log"],
+        topsrcdir=topsrcdir,
+        topobjdir=topobjdir,
+        tasks=lintargs["substs"]["GRADLE_ANDROID_CHECKSTYLE_TASKS"],
+        extra_args=lintargs.get("extra_args") or [],
+    )
+
+    results = []
+
+    for relative_path in lintargs["substs"]["GRADLE_ANDROID_CHECKSTYLE_OUTPUT_FILES"]:
+        report_path = os.path.join(lintargs["topobjdir"], relative_path)
+        results.extend(
+            _parse_checkstyle_output(
+                config, topsrcdir=lintargs["root"], report_path=report_path
+            )
+        )
+
+    return results
+
+
+def _parse_android_test_results(config, topsrcdir=None, report_dir=None):
+    # A brute force way to turn a Java FQN into a path on disk.  Assumes Java
+    # and Kotlin sources are in mobile/android for performance and simplicity.
+    sourcepath_finder = FileFinder(os.path.join(topsrcdir, "mobile", "android"))
+
+    finder = FileFinder(report_dir)
+    reports = list(finder.find("TEST-*.xml"))
+    if not reports:
+        raise RuntimeError("No reports found under {}".format(report_dir))
+
+    for report, _ in reports:
+        tree = ET.parse(open(os.path.join(finder.base, report), "rt"))
+        root = tree.getroot()
+
+        class_name = root.get(
+            "name"
+        )  # Like 'org.mozilla.gecko.permissions.TestPermissions'.
+        path = (
+            "**/" + class_name.replace(".", "/") + ".*"
+        )  # Like '**/org/mozilla/gecko/permissions/TestPermissions.*'.  # NOQA: E501
+
+        for testcase in root.findall("testcase"):
+            function_name = testcase.get("name")
+
+            # Schema cribbed from http://llg.cubic.org/docs/junit/.
+            for unexpected in itertools.chain(
+                testcase.findall("error"), testcase.findall("failure")
+            ):
+                sourcepaths = list(sourcepath_finder.find(path))
+                if not sourcepaths:
+                    raise RuntimeError(
+                        "No sourcepath found for class {class_name}".format(
+                            class_name=class_name
+                        )
+                    )
+
+                for sourcepath, _ in sourcepaths:
+                    lineno = 0
+                    message = unexpected.get("message")
+                    # Turn '... at org.mozilla.gecko.permissions.TestPermissions.testMultipleRequestsAreQueuedAndDispatchedSequentially(TestPermissions.java:118)' into 118.  # NOQA: E501
+                    pattern = r"at {class_name}\.{function_name}\(.*:(\d+)\)"
+                    pattern = pattern.format(
+                        class_name=class_name, function_name=function_name
+                    )
+                    match = re.search(pattern, message)
+                    if match:
+                        lineno = int(match.group(1))
+                    else:
+                        msg = "No source line found for {class_name}.{function_name}".format(
+                            class_name=class_name, function_name=function_name
+                        )
+                        raise RuntimeError(msg)
+
+                    err = {
+                        "level": "error",
+                        "rule": unexpected.get("type"),
+                        "message": message,
+                        "path": os.path.join(
+                            topsrcdir, "mobile", "android", sourcepath
+                        ),
+                        "lineno": lineno,
+                    }
+                    yield result.from_config(config, **err)
+
+
+def test(config, **lintargs):
+    topsrcdir = lintargs["root"]
+    topobjdir = lintargs["topobjdir"]
+
+    gradle(
+        lintargs["log"],
+        topsrcdir=topsrcdir,
+        topobjdir=topobjdir,
+        tasks=lintargs["substs"]["GRADLE_ANDROID_TEST_TASKS"],
+        extra_args=lintargs.get("extra_args") or [],
+    )
+
+    results = []
+
+    def capitalize(s):
+        # Can't use str.capitalize because it lower cases trailing letters.
+        return (s[0].upper() + s[1:]) if s else ""
+
+    pairs = [("geckoview", lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_VARIANT_NAME"])]
+    for project, variant in pairs:
+        report_dir = os.path.join(
+            lintargs["topobjdir"],
+            "gradle/build/mobile/android/{}/test-results/test{}UnitTest".format(
+                project, capitalize(variant)
+            ),
+        )
+    results.extend(
+        _parse_android_test_results(
+            config, topsrcdir=lintargs["root"], report_dir=report_dir
+        )
+    )
+
+    return results