Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 470 insertions, 0 deletions

diff --git a/tools/lint/fluent-lint/__init__.py b/tools/lint/fluent-lint/__init__.py
new file mode 100644
index 0000000000..3d0373ea01
--- /dev/null
+++ b/tools/lint/fluent-lint/__init__.py
@@ -0,0 +1,470 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import bisect
+import os
+import re
+from html.parser import HTMLParser
+
+import mozpack.path as mozpath
+import yaml
+from fluent.syntax import ast, parse, visitor
+from mozlint import result
+from mozlint.pathutils import expand_exclusions
+
+
+class TextElementHTMLParser(HTMLParser):
+    """HTML Parser for TextElement.
+
+    TextElements may contain embedded html tags, which can include
+    quotes in attributes. We only want to check the actual text.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.extracted_text = []
+
+    def handle_data(self, data):
+        self.extracted_text.append(data)
+
+
+class Linter(visitor.Visitor):
+    """Fluent linter implementation.
+
+    This subclasses the Fluent AST visitor. Methods are called corresponding
+    to each type of node in the Fluent AST. It is possible to control
+    whether a node is recursed into by calling the generic_visit method on
+    the superclass.
+
+    See the documentation here:
+    https://www.projectfluent.org/python-fluent/fluent.syntax/stable/usage.html
+    """
+
+    def __init__(
+        self, path, config, exclusions, contents, offsets_and_lines, brand_names=[]
+    ):
+        super().__init__()
+        self.path = path
+        self.config = config
+        self.exclusions = exclusions
+        self.contents = contents
+        self.offsets_and_lines = offsets_and_lines
+
+        self.results = []
+        self.identifier_re = re.compile(r"[a-z0-9-]+")
+        self.apostrophe_re = re.compile(r"\w'")
+        self.incorrect_apostrophe_re = re.compile(r"\w\u2018\w")
+        self.single_quote_re = re.compile(r"'(.+)'")
+        self.double_quote_re = re.compile(r"\".+\"")
+        self.ellipsis_re = re.compile(r"\.\.\.")
+
+        self.brand_names = brand_names
+        self.minimum_id_length = 9
+
+        self.state = {
+            # The resource comment should be at the top of the page after the license.
+            "node_can_be_resource_comment": True,
+            # Group comments must be followed by a message. Two group comments are not
+            # allowed in a row.
+            "can_have_group_comment": True,
+            # Comment bound to the current message
+            "comment": "",
+            # The current group comment
+            "group_comment": "",
+            # Variables in the current message
+            "variables": [],
+        }
+
+        # Set this to true to debug print the root node's json. This is useful for
+        # writing new lint rules, or debugging existing ones.
+        self.debug_print_json = False
+
+    def generic_visit(self, node):
+        node_name = type(node).__name__
+        self.state["node_can_be_resource_comment"] = self.state[
+            "node_can_be_resource_comment"
+        ] and (
+            # This is the root node.
+            node_name == "Resource"
+            # Empty space is allowed.
+            or node_name == "Span"
+            # Comments are allowed
+            or node_name == "Comment"
+        )
+
+        if self.debug_print_json:
+            import json
+
+            print(json.dumps(node.to_json(), indent=2))
+            # Only debug print the root node.
+            self.debug_print_json = False
+
+        super(Linter, self).generic_visit(node)
+
+    def visit_Attribute(self, node):
+        # Only visit values for Attribute nodes, the identifier comes from dom.
+        super().generic_visit(node.value)
+
+    def visit_FunctionReference(self, node):
+        # We don't recurse into function references, the identifiers there are
+        # allowed to be free form.
+        pass
+
+    def visit_Message(self, node):
+        # There must be at least one message or term between group comments.
+        self.state["can_have_group_comment"] = True
+        self.last_message_id = node.id.name
+
+        super().generic_visit(node)
+
+        # Check if variables are referenced in comments
+        if self.state["variables"]:
+            comments = self.state["comment"] + self.state["group_comment"]
+            missing_references = [
+                v for v in self.state["variables"] if f"${v}" not in comments
+            ]
+            if missing_references:
+                self.add_error(
+                    node,
+                    "VC01",
+                    "Messages including variables should have a comment "
+                    "explaining what will replace the variable. "
+                    "Missing references: "
+                    + ", ".join([f"${m}" for m in missing_references]),
+                )
+
+        # Reset current comment and variable references after reading the
+        # message.
+        self.state["comment"] = ""
+        self.state["variables"] = []
+
+    def visit_Term(self, node):
+        # There must be at least one message or term between group comments.
+        self.state["can_have_group_comment"] = True
+        self.last_message_id = None
+
+        super().generic_visit(node)
+
+        # Reset current comment and variable references after reading the term.
+        self.state["comment"] = ""
+        self.state["variables"] = []
+
+    def visit_MessageReference(self, node):
+        # We don't recurse into message references, the identifiers are either
+        # checked elsewhere or are attributes and come from DOM.
+        pass
+
+    def visit_Identifier(self, node):
+        if (
+            self.path not in self.exclusions["ID01"]["files"]
+            and node.name not in self.exclusions["ID01"]["messages"]
+            and not self.identifier_re.fullmatch(node.name)
+        ):
+            self.add_error(
+                node,
+                "ID01",
+                "Identifiers may only contain lowercase characters and -",
+            )
+        if (
+            len(node.name) < self.minimum_id_length
+            and self.path not in self.exclusions["ID02"]["files"]
+            and node.name not in self.exclusions["ID02"]["messages"]
+        ):
+            self.add_error(
+                node,
+                "ID02",
+                f"Identifiers must be at least {self.minimum_id_length} characters long",
+            )
+
+    def visit_TextElement(self, node):
+        parser = TextElementHTMLParser()
+        parser.feed(node.value)
+        for text in parser.extracted_text:
+            # To check for apostrophes, first remove pairs of straight quotes
+            # used as delimiters.
+            cleaned_str = re.sub(self.single_quote_re, "\1", node.value)
+            if self.apostrophe_re.search(cleaned_str):
+                self.add_error(
+                    node,
+                    "TE01",
+                    "Strings with apostrophes should use foo\u2019s instead of foo's.",
+                )
+            if self.incorrect_apostrophe_re.search(text):
+                self.add_error(
+                    node,
+                    "TE02",
+                    "Strings with apostrophes should use foo\u2019s instead of foo\u2018s.",
+                )
+            if self.single_quote_re.search(text):
+                self.add_error(
+                    node,
+                    "TE03",
+                    "Single-quoted strings should use Unicode \u2018foo\u2019 instead of 'foo'.",
+                )
+            if self.double_quote_re.search(text):
+                self.add_error(
+                    node,
+                    "TE04",
+                    'Double-quoted strings should use Unicode \u201cfoo\u201d instead of "foo".',
+                )
+            if self.ellipsis_re.search(text):
+                self.add_error(
+                    node,
+                    "TE05",
+                    "Strings with an ellipsis should use the Unicode \u2026 character"
+                    " instead of three periods",
+                )
+
+            # If part of a message, check for brand names
+            if (
+                self.last_message_id is not None
+                and self.path not in self.exclusions["CO01"]["files"]
+                and self.last_message_id not in self.exclusions["CO01"]["messages"]
+            ):
+                found_brands = []
+                for brand in self.brand_names:
+                    if brand in text:
+                        found_brands.append(brand)
+                if found_brands:
+                    self.add_error(
+                        node,
+                        "CO01",
+                        "Strings should use the corresponding terms instead of"
+                        f" hard-coded brand names ({', '.join(found_brands)})",
+                    )
+
+    def visit_ResourceComment(self, node):
+        # This node is a comment with: "###"
+        if not self.state["node_can_be_resource_comment"]:
+            self.add_error(
+                node,
+                "RC01",
+                "Resource comments (###) should be placed at the top of the file, just "
+                "after the license header. There should only be one resource comment "
+                "per file.",
+            )
+            return
+
+        lines_after = get_newlines_count_after(node.span, self.contents)
+        lines_before = get_newlines_count_before(node.span, self.contents)
+
+        if node.span.end == len(self.contents) - 1:
+            # This file only contains a resource comment.
+            return
+
+        if lines_after != 2:
+            self.add_error(
+                node,
+                "RC02",
+                "Resource comments (###) should be followed by one empty line.",
+            )
+            return
+
+        if lines_before != 2:
+            self.add_error(
+                node,
+                "RC03",
+                "Resource comments (###) should have one empty line above them.",
+            )
+            return
+
+    def visit_SelectExpression(self, node):
+        # We only want to visit the variant values, the identifiers in selectors
+        # and keys are allowed to be free form.
+        for variant in node.variants:
+            super().generic_visit(variant.value)
+
+        # Store the variable used for the SelectExpression, excluding functions
+        # like PLATFORM()
+        if (
+            type(node.selector) == ast.VariableReference
+            and node.selector.id.name not in self.state["variables"]
+        ):
+            self.state["variables"].append(node.selector.id.name)
+
+    def visit_Comment(self, node):
+        # This node is a comment with: "#"
+
+        # Store the comment
+        self.state["comment"] = node.content
+
+    def visit_GroupComment(self, node):
+        # This node is a comment with: "##"
+
+        # Store the group comment
+        self.state["group_comment"] = node.content
+
+        if not self.state["can_have_group_comment"]:
+            self.add_error(
+                node,
+                "GC04",
+                "Group comments (##) must be followed by at least one message "
+                "or term. Make sure that a single group comment with multiple "
+                "paragraphs is not separated by whitespace, as it will be "
+                "interpreted as two different comments.",
+            )
+            return
+
+        self.state["can_have_group_comment"] = False
+
+        lines_after = get_newlines_count_after(node.span, self.contents)
+        lines_before = get_newlines_count_before(node.span, self.contents)
+
+        if node.span.end == len(self.contents) - 1:
+            # The group comment is the last thing in the file.
+
+            if node.content == "":
+                # Empty comments are allowed at the end of the file.
+                return
+
+            self.add_error(
+                node,
+                "GC01",
+                "Group comments (##) should not be at the end of the file, they should "
+                "always be above a message. Only an empty group comment is allowed at "
+                "the end of a file.",
+            )
+            return
+
+        if lines_after != 2:
+            self.add_error(
+                node,
+                "GC02",
+                "Group comments (##) should be followed by one empty line.",
+            )
+            return
+
+        if lines_before != 2:
+            self.add_error(
+                node,
+                "GC03",
+                "Group comments (##) should have an empty line before them.",
+            )
+            return
+
+    def visit_VariableReference(self, node):
+        # Identifiers are allowed to be free form, but need to store them
+        # for comment checks.
+
+        if node.id.name not in self.state["variables"]:
+            self.state["variables"].append(node.id.name)
+
+    def add_error(self, node, rule, msg):
+        (col, line) = self.span_to_line_and_col(node.span)
+        res = {
+            "path": self.path,
+            "lineno": line,
+            "column": col,
+            "rule": rule,
+            "message": msg,
+        }
+        self.results.append(result.from_config(self.config, **res))
+
+    def span_to_line_and_col(self, span):
+        i = bisect.bisect_left(self.offsets_and_lines, (span.start, 0))
+        if i > 0:
+            col = span.start - self.offsets_and_lines[i - 1][0]
+        else:
+            col = 1 + span.start
+        return (col, self.offsets_and_lines[i][1])
+
+
+def get_offsets_and_lines(contents):
+    """Return a list consisting of tuples of (offset, line).
+
+    The Fluent AST contains spans of start and end offsets in the file.
+    This function returns a list of offsets and line numbers so that errors
+    can be reported using line and column.
+    """
+    line = 1
+    result = []
+    for m in re.finditer(r"\n", contents):
+        result.append((m.start(), line))
+        line += 1
+    return result
+
+
+def get_newlines_count_after(span, contents):
+    # Determine the number of newlines.
+    count = 0
+    for i in range(span.end, len(contents)):
+        assert contents[i] != "\r", "This linter does not handle \\r characters."
+        if contents[i] != "\n":
+            break
+        count += 1
+
+    return count
+
+
+def get_newlines_count_before(span, contents):
+    # Determine the range of newline characters.
+    count = 0
+    for i in range(span.start - 1, 0, -1):
+        assert contents[i] != "\r", "This linter does not handle \\r characters."
+        if contents[i] != "\n":
+            break
+        count += 1
+
+    return count
+
+
+def get_exclusions(root):
+    with open(
+        mozpath.join(root, "tools", "lint", "fluent-lint", "exclusions.yml")
+    ) as f:
+        exclusions = list(yaml.safe_load_all(f))[0]
+        for error_type in exclusions:
+            exclusions[error_type]["files"] = set(
+                [mozpath.join(root, x) for x in exclusions[error_type]["files"]]
+            )
+        return exclusions
+
+
+def get_branding_list(root, brand_files):
+    class MessageExtractor(visitor.Visitor):
+        def __init__(self):
+            self.brands = []
+            self.last_message_id = None
+
+        def visit_Term(self, node):
+            self.last_message_id = node.id.name
+            self.generic_visit(node)
+
+        def visit_TextElement(self, node):
+            if self.last_message_id:
+                self.brands += [node.value]
+                self.last_message_id = None
+            self.generic_visit(node)
+
+    extractor = MessageExtractor()
+
+    for brand_path in brand_files:
+        brand_file = mozpath.join(root, brand_path)
+        if os.path.exists(brand_file):
+            with open(brand_file, encoding="utf-8") as f:
+                messages = parse(f.read())
+                extractor.visit(messages)
+
+    return list(set(extractor.brands))
+
+
+def lint(paths, config, fix=None, **lintargs):
+    root = lintargs["root"]
+    files = list(expand_exclusions(paths, config, root))
+    exclusions = get_exclusions(root)
+    brand_files = config.get("brand-files")
+    brand_names = get_branding_list(root, brand_files)
+    results = []
+    for path in files:
+        contents = open(path, "r", encoding="utf-8").read()
+        linter = Linter(
+            path,
+            config,
+            exclusions,
+            contents,
+            get_offsets_and_lines(contents),
+            brand_names,
+        )
+        linter.visit(parse(contents))
+        results.extend(linter.results)
+    return results