Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

2 files changed, 668 insertions, 0 deletions

diff --git a/tools/lint/fluent-lint/__init__.py b/tools/lint/fluent-lint/__init__.py
new file mode 100644
index 0000000000..3d0373ea01
--- /dev/null
+++ b/tools/lint/fluent-lint/__init__.py
@@ -0,0 +1,470 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import bisect
+import os
+import re
+from html.parser import HTMLParser
+
+import mozpack.path as mozpath
+import yaml
+from fluent.syntax import ast, parse, visitor
+from mozlint import result
+from mozlint.pathutils import expand_exclusions
+
+
+class TextElementHTMLParser(HTMLParser):
+    """HTML Parser for TextElement.
+
+    TextElements may contain embedded html tags, which can include
+    quotes in attributes. We only want to check the actual text.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.extracted_text = []
+
+    def handle_data(self, data):
+        self.extracted_text.append(data)
+
+
+class Linter(visitor.Visitor):
+    """Fluent linter implementation.
+
+    This subclasses the Fluent AST visitor. Methods are called corresponding
+    to each type of node in the Fluent AST. It is possible to control
+    whether a node is recursed into by calling the generic_visit method on
+    the superclass.
+
+    See the documentation here:
+    https://www.projectfluent.org/python-fluent/fluent.syntax/stable/usage.html
+    """
+
+    def __init__(
+        self, path, config, exclusions, contents, offsets_and_lines, brand_names=[]
+    ):
+        super().__init__()
+        self.path = path
+        self.config = config
+        self.exclusions = exclusions
+        self.contents = contents
+        self.offsets_and_lines = offsets_and_lines
+
+        self.results = []
+        self.identifier_re = re.compile(r"[a-z0-9-]+")
+        self.apostrophe_re = re.compile(r"\w'")
+        self.incorrect_apostrophe_re = re.compile(r"\w\u2018\w")
+        self.single_quote_re = re.compile(r"'(.+)'")
+        self.double_quote_re = re.compile(r"\".+\"")
+        self.ellipsis_re = re.compile(r"\.\.\.")
+
+        self.brand_names = brand_names
+        self.minimum_id_length = 9
+
+        self.state = {
+            # The resource comment should be at the top of the page after the license.
+            "node_can_be_resource_comment": True,
+            # Group comments must be followed by a message. Two group comments are not
+            # allowed in a row.
+            "can_have_group_comment": True,
+            # Comment bound to the current message
+            "comment": "",
+            # The current group comment
+            "group_comment": "",
+            # Variables in the current message
+            "variables": [],
+        }
+
+        # Set this to true to debug print the root node's json. This is useful for
+        # writing new lint rules, or debugging existing ones.
+        self.debug_print_json = False
+
+    def generic_visit(self, node):
+        node_name = type(node).__name__
+        self.state["node_can_be_resource_comment"] = self.state[
+            "node_can_be_resource_comment"
+        ] and (
+            # This is the root node.
+            node_name == "Resource"
+            # Empty space is allowed.
+            or node_name == "Span"
+            # Comments are allowed
+            or node_name == "Comment"
+        )
+
+        if self.debug_print_json:
+            import json
+
+            print(json.dumps(node.to_json(), indent=2))
+            # Only debug print the root node.
+            self.debug_print_json = False
+
+        super(Linter, self).generic_visit(node)
+
+    def visit_Attribute(self, node):
+        # Only visit values for Attribute nodes, the identifier comes from dom.
+        super().generic_visit(node.value)
+
+    def visit_FunctionReference(self, node):
+        # We don't recurse into function references, the identifiers there are
+        # allowed to be free form.
+        pass
+
+    def visit_Message(self, node):
+        # There must be at least one message or term between group comments.
+        self.state["can_have_group_comment"] = True
+        self.last_message_id = node.id.name
+
+        super().generic_visit(node)
+
+        # Check if variables are referenced in comments
+        if self.state["variables"]:
+            comments = self.state["comment"] + self.state["group_comment"]
+            missing_references = [
+                v for v in self.state["variables"] if f"${v}" not in comments
+            ]
+            if missing_references:
+                self.add_error(
+                    node,
+                    "VC01",
+                    "Messages including variables should have a comment "
+                    "explaining what will replace the variable. "
+                    "Missing references: "
+                    + ", ".join([f"${m}" for m in missing_references]),
+                )
+
+        # Reset current comment and variable references after reading the
+        # message.
+        self.state["comment"] = ""
+        self.state["variables"] = []
+
+    def visit_Term(self, node):
+        # There must be at least one message or term between group comments.
+        self.state["can_have_group_comment"] = True
+        self.last_message_id = None
+
+        super().generic_visit(node)
+
+        # Reset current comment and variable references after reading the term.
+        self.state["comment"] = ""
+        self.state["variables"] = []
+
+    def visit_MessageReference(self, node):
+        # We don't recurse into message references, the identifiers are either
+        # checked elsewhere or are attributes and come from DOM.
+        pass
+
+    def visit_Identifier(self, node):
+        if (
+            self.path not in self.exclusions["ID01"]["files"]
+            and node.name not in self.exclusions["ID01"]["messages"]
+            and not self.identifier_re.fullmatch(node.name)
+        ):
+            self.add_error(
+                node,
+                "ID01",
+                "Identifiers may only contain lowercase characters and -",
+            )
+        if (
+            len(node.name) < self.minimum_id_length
+            and self.path not in self.exclusions["ID02"]["files"]
+            and node.name not in self.exclusions["ID02"]["messages"]
+        ):
+            self.add_error(
+                node,
+                "ID02",
+                f"Identifiers must be at least {self.minimum_id_length} characters long",
+            )
+
+    def visit_TextElement(self, node):
+        parser = TextElementHTMLParser()
+        parser.feed(node.value)
+        for text in parser.extracted_text:
+            # To check for apostrophes, first remove pairs of straight quotes
+            # used as delimiters.
+            cleaned_str = re.sub(self.single_quote_re, "\1", node.value)
+            if self.apostrophe_re.search(cleaned_str):
+                self.add_error(
+                    node,
+                    "TE01",
+                    "Strings with apostrophes should use foo\u2019s instead of foo's.",
+                )
+            if self.incorrect_apostrophe_re.search(text):
+                self.add_error(
+                    node,
+                    "TE02",
+                    "Strings with apostrophes should use foo\u2019s instead of foo\u2018s.",
+                )
+            if self.single_quote_re.search(text):
+                self.add_error(
+                    node,
+                    "TE03",
+                    "Single-quoted strings should use Unicode \u2018foo\u2019 instead of 'foo'.",
+                )
+            if self.double_quote_re.search(text):
+                self.add_error(
+                    node,
+                    "TE04",
+                    'Double-quoted strings should use Unicode \u201cfoo\u201d instead of "foo".',
+                )
+            if self.ellipsis_re.search(text):
+                self.add_error(
+                    node,
+                    "TE05",
+                    "Strings with an ellipsis should use the Unicode \u2026 character"
+                    " instead of three periods",
+                )
+
+            # If part of a message, check for brand names
+            if (
+                self.last_message_id is not None
+                and self.path not in self.exclusions["CO01"]["files"]
+                and self.last_message_id not in self.exclusions["CO01"]["messages"]
+            ):
+                found_brands = []
+                for brand in self.brand_names:
+                    if brand in text:
+                        found_brands.append(brand)
+                if found_brands:
+                    self.add_error(
+                        node,
+                        "CO01",
+                        "Strings should use the corresponding terms instead of"
+                        f" hard-coded brand names ({', '.join(found_brands)})",
+                    )
+
+    def visit_ResourceComment(self, node):
+        # This node is a comment with: "###"
+        if not self.state["node_can_be_resource_comment"]:
+            self.add_error(
+                node,
+                "RC01",
+                "Resource comments (###) should be placed at the top of the file, just "
+                "after the license header. There should only be one resource comment "
+                "per file.",
+            )
+            return
+
+        lines_after = get_newlines_count_after(node.span, self.contents)
+        lines_before = get_newlines_count_before(node.span, self.contents)
+
+        if node.span.end == len(self.contents) - 1:
+            # This file only contains a resource comment.
+            return
+
+        if lines_after != 2:
+            self.add_error(
+                node,
+                "RC02",
+                "Resource comments (###) should be followed by one empty line.",
+            )
+            return
+
+        if lines_before != 2:
+            self.add_error(
+                node,
+                "RC03",
+                "Resource comments (###) should have one empty line above them.",
+            )
+            return
+
+    def visit_SelectExpression(self, node):
+        # We only want to visit the variant values, the identifiers in selectors
+        # and keys are allowed to be free form.
+        for variant in node.variants:
+            super().generic_visit(variant.value)
+
+        # Store the variable used for the SelectExpression, excluding functions
+        # like PLATFORM()
+        if (
+            type(node.selector) == ast.VariableReference
+            and node.selector.id.name not in self.state["variables"]
+        ):
+            self.state["variables"].append(node.selector.id.name)
+
+    def visit_Comment(self, node):
+        # This node is a comment with: "#"
+
+        # Store the comment
+        self.state["comment"] = node.content
+
+    def visit_GroupComment(self, node):
+        # This node is a comment with: "##"
+
+        # Store the group comment
+        self.state["group_comment"] = node.content
+
+        if not self.state["can_have_group_comment"]:
+            self.add_error(
+                node,
+                "GC04",
+                "Group comments (##) must be followed by at least one message "
+                "or term. Make sure that a single group comment with multiple "
+                "paragraphs is not separated by whitespace, as it will be "
+                "interpreted as two different comments.",
+            )
+            return
+
+        self.state["can_have_group_comment"] = False
+
+        lines_after = get_newlines_count_after(node.span, self.contents)
+        lines_before = get_newlines_count_before(node.span, self.contents)
+
+        if node.span.end == len(self.contents) - 1:
+            # The group comment is the last thing in the file.
+
+            if node.content == "":
+                # Empty comments are allowed at the end of the file.
+                return
+
+            self.add_error(
+                node,
+                "GC01",
+                "Group comments (##) should not be at the end of the file, they should "
+                "always be above a message. Only an empty group comment is allowed at "
+                "the end of a file.",
+            )
+            return
+
+        if lines_after != 2:
+            self.add_error(
+                node,
+                "GC02",
+                "Group comments (##) should be followed by one empty line.",
+            )
+            return
+
+        if lines_before != 2:
+            self.add_error(
+                node,
+                "GC03",
+                "Group comments (##) should have an empty line before them.",
+            )
+            return
+
+    def visit_VariableReference(self, node):
+        # Identifiers are allowed to be free form, but need to store them
+        # for comment checks.
+
+        if node.id.name not in self.state["variables"]:
+            self.state["variables"].append(node.id.name)
+
+    def add_error(self, node, rule, msg):
+        (col, line) = self.span_to_line_and_col(node.span)
+        res = {
+            "path": self.path,
+            "lineno": line,
+            "column": col,
+            "rule": rule,
+            "message": msg,
+        }
+        self.results.append(result.from_config(self.config, **res))
+
+    def span_to_line_and_col(self, span):
+        i = bisect.bisect_left(self.offsets_and_lines, (span.start, 0))
+        if i > 0:
+            col = span.start - self.offsets_and_lines[i - 1][0]
+        else:
+            col = 1 + span.start
+        return (col, self.offsets_and_lines[i][1])
+
+
+def get_offsets_and_lines(contents):
+    """Return a list consisting of tuples of (offset, line).
+
+    The Fluent AST contains spans of start and end offsets in the file.
+    This function returns a list of offsets and line numbers so that errors
+    can be reported using line and column.
+    """
+    line = 1
+    result = []
+    for m in re.finditer(r"\n", contents):
+        result.append((m.start(), line))
+        line += 1
+    return result
+
+
+def get_newlines_count_after(span, contents):
+    # Determine the number of newlines.
+    count = 0
+    for i in range(span.end, len(contents)):
+        assert contents[i] != "\r", "This linter does not handle \\r characters."
+        if contents[i] != "\n":
+            break
+        count += 1
+
+    return count
+
+
+def get_newlines_count_before(span, contents):
+    # Determine the range of newline characters.
+    count = 0
+    for i in range(span.start - 1, 0, -1):
+        assert contents[i] != "\r", "This linter does not handle \\r characters."
+        if contents[i] != "\n":
+            break
+        count += 1
+
+    return count
+
+
+def get_exclusions(root):
+    with open(
+        mozpath.join(root, "tools", "lint", "fluent-lint", "exclusions.yml")
+    ) as f:
+        exclusions = list(yaml.safe_load_all(f))[0]
+        for error_type in exclusions:
+            exclusions[error_type]["files"] = set(
+                [mozpath.join(root, x) for x in exclusions[error_type]["files"]]
+            )
+        return exclusions
+
+
+def get_branding_list(root, brand_files):
+    class MessageExtractor(visitor.Visitor):
+        def __init__(self):
+            self.brands = []
+            self.last_message_id = None
+
+        def visit_Term(self, node):
+            self.last_message_id = node.id.name
+            self.generic_visit(node)
+
+        def visit_TextElement(self, node):
+            if self.last_message_id:
+                self.brands += [node.value]
+                self.last_message_id = None
+            self.generic_visit(node)
+
+    extractor = MessageExtractor()
+
+    for brand_path in brand_files:
+        brand_file = mozpath.join(root, brand_path)
+        if os.path.exists(brand_file):
+            with open(brand_file, encoding="utf-8") as f:
+                messages = parse(f.read())
+                extractor.visit(messages)
+
+    return list(set(extractor.brands))
+
+
+def lint(paths, config, fix=None, **lintargs):
+    root = lintargs["root"]
+    files = list(expand_exclusions(paths, config, root))
+    exclusions = get_exclusions(root)
+    brand_files = config.get("brand-files")
+    brand_names = get_branding_list(root, brand_files)
+    results = []
+    for path in files:
+        contents = open(path, "r", encoding="utf-8").read()
+        linter = Linter(
+            path,
+            config,
+            exclusions,
+            contents,
+            get_offsets_and_lines(contents),
+            brand_names,
+        )
+        linter.visit(parse(contents))
+        results.extend(linter.results)
+    return results
diff --git a/tools/lint/fluent-lint/exclusions.yml b/tools/lint/fluent-lint/exclusions.yml
new file mode 100644
index 0000000000..ebcd1b0029
--- /dev/null
+++ b/tools/lint/fluent-lint/exclusions.yml
@@ -0,0 +1,198 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Warning: Only exclusions for identifiers (ID01) are currently allowed.
+---
+# Only add exceptions to this file if the ID is generated programmatically and
+# can't easily be changed to follow the naming convention.
+# Only lowercase letters and hyphens should be used in Fluent IDs.
+ID01:
+    messages:
+        - trademarkInfo
+        - crashed-include-URL-2
+        - blocklist-item-moz-std-listName
+        - blocklist-item-moz-full-listName
+        - shortcuts-browserAction2
+        - shortcuts-pageAction
+        - shortcuts-sidebarAction
+        - about-networking-originAttributesSuffix
+        - size-KB
+        - size-MB
+        - size-GB
+        - state-dd-Disabled
+        - state-dd-Disabled-block-list-state
+        - memory-unit-B
+        - memory-unit-KB
+        - memory-unit-MB
+        - memory-unit-GB
+        - memory-unit-TB
+        - memory-unit-PB
+        - memory-unit-EB
+        - enableSafeBrowsing-label
+        - about-telemetry-show-in-Firefox-json-viewer
+        - url-classifier-search-listType
+        # aboutDialog.ftl: Do not add new exceptions for this file,
+        # new strings should follow the naming convention.
+        - aboutDialog-title
+        - releaseNotes-link
+        - update-checkForUpdatesButton
+        - update-updateButton
+        - update-checkingForUpdates
+        - update-adminDisabled
+        - update-noUpdatesFound
+        - update-otherInstanceHandlingUpdates
+        - warningDesc-version
+        - bottomLinks-license
+        - bottomLinks-rights
+        - bottomLinks-privacy
+        - aboutDialog-version
+        - aboutDialog-version-nightly
+        # certError.ftl: These IDs are generated programmatically
+        # from certificate error codes.
+        - connectionFailure-title
+        - deniedPortAccess-title
+        - dnsNotFound-title
+        - fileNotFound-title
+        - fileAccessDenied-title
+        - captivePortal-title
+        - malformedURI-title
+        - netInterrupt-title
+        - notCached-title
+        - netOffline-title
+        - contentEncodingError-title
+        - unsafeContentType-title
+        - netReset-title
+        - netTimeout-title
+        - unknownProtocolFound-title
+        - proxyConnectFailure-title
+        - proxyResolveFailure-title
+        - redirectLoop-title
+        - unknownSocketType-title
+        - nssFailure2-title
+        - corruptedContentError-title
+        - sslv3Used-title
+        - inadequateSecurityError-title
+        - blockedByPolicy-title
+        - clockSkewError-title
+        - networkProtocolError-title
+        - nssBadCert-title
+        - nssBadCert-sts-title
+    files:
+        # policies-descriptions.ftl: These IDs are generated programmatically
+        # from policy names.
+        - browser/locales/en-US/browser/policies/policies-descriptions.ftl
+        # The webext-perms-description-* IDs are generated programmatically
+        # from permission names
+        - toolkit/locales/en-US/toolkit/global/extensionPermissions.ftl
+ID02:
+    messages:
+        # browser/components/ion/content/ion.ftl
+        - ion
+        # browser/locales/en-US/browser/aboutDialog.ftl
+        - helpus
+        # browser/locales/en-US/browser/aboutLogins.ftl
+        - menu
+        # browser/locales/en-US/browser/pageInfo.ftl
+        - copy
+        - perm-tab
+        # browser/locales/en-US/browser/tabContextMenu.ftl
+        - pin-tab
+        # browser/locales/en-US/browser/touchbar/touchbar.ftl
+        - back
+        - forward
+        - reload
+        - home
+        - find
+        - new-tab
+        - share
+        # toolkit/locales/en-US/toolkit/about/aboutPerformance.ftl
+        - type-tab
+        - size-KB
+        - size-MB
+        - size-GB
+        - item
+        # toolkit/locales/en-US/toolkit/about/aboutPlugins.ftl
+        - file-dd
+        - path-dd
+        # toolkit/locales/en-US/toolkit/about/aboutServiceWorkers.ftl
+        - scope
+        - waiting
+        # toolkit/locales/en-US/toolkit/about/aboutSupport.ftl
+        # yaml interprets yes and no as booleans if quotes are not present.
+        - "yes"
+        - "no"
+        - unknown
+        - found
+        - missing
+        - gpu-ram
+        - apz-none
+        # toolkit/locales/en-US/toolkit/printing/printDialogs.ftl
+        - portrait
+        - scale
+        - print-bg
+        - hf-blank
+        - hf-title
+        - hf-url
+        - hf-page
+    files: []
+# Hard-coded brand names like Firefox or Mozilla should be used only in
+# specific cases, in all other cases the corresponding terms should be used.
+# Check with the localization team for advice.
+CO01:
+    messages:
+        # browser/branding/official/locales/en-US/brand.ftl
+        - trademarkInfo
+        # toolkit/locales/en-US/toolkit/neterror/certError.ftl
+        - cert-error-mitm-mozilla
+        - cert-error-mitm-connection
+        # browser/locales/en-US/browser/appExtensionFields.ftl
+        - extension-firefox-alpenglow-name
+        # browser/locales/en-US/browser/browser.ftl
+        - identity-custom-root
+        - identity-description-custom-root2
+        # browser/locales/en-US/browser/migration.ftl
+        - import-from-firefox
+        # browser/locales/en-US/browser/migrationWizard.ftl
+        - migration-wizard-migrator-display-name-firefox
+        # browser/locales/en-US/browser/newtab/onboarding.ftl
+        - mr1-onboarding-welcome-image-caption
+        - mr2022-onboarding-gratitude-subtitle
+        # browser/locales/en-US/browser/policies/policies-descriptions.ftl
+        - policy-DisableFirefoxScreenshots
+        # browser/locales/en-US/browser/preferences/preferences.ftl
+        - sync-engine-addons
+        - sync-mobile-promo
+        # browser/locales/en-US/browser/protectionsPanel.ftl
+        - protections-panel-content-blocking-breakage-report-view-description
+        # devtools/client/locales/en-US/aboutdebugging.ftl
+        - about-debugging-setup-usb-step-enable-debug-firefox2
+        - about-debugging-browser-version-too-old-fennec
+        - about-debugging-browser-version-too-recent
+        # devtools/client/locales/en-US/application.ftl
+        - manifest-loaded-devtools-error
+        # toolkit/locales/en-US/toolkit/about/aboutAddons.ftl
+        - addon-badge-line3
+        - recommended-theme-1
+        # toolkit/locales/en-US/toolkit/about/aboutGlean.ftl
+        - about-glean-description
+        # toolkit/locales/en-US/toolkit/about/aboutPlugins.ftl
+        - plugins-openh264-description
+        # toolkit/locales/en-US/toolkit/about/aboutRights.ftl
+        - rights-intro-point-1
+        - rights-intro-point-2
+        # toolkit/locales/en-US/toolkit/about/aboutSupport.ftl
+        - app-basics-key-mozilla
+        - virtual-monitor-disp
+        # toolkit/locales/en-US/toolkit/about/aboutTelemetry.ftl
+        - about-telemetry-firefox-data-doc
+        - about-telemetry-telemetry-client-doc
+        - about-telemetry-telemetry-dashboard
+        # toolkit/locales/en-US/toolkit/global/extensionPermissions.ftl
+        - webext-perms-description-management
+        # toolkit/locales/en-US/toolkit/global/processTypes.ftl
+        - process-type-privilegedmozilla
+    files:
+        - browser/components/ion/content/ion.ftl
+        - browser/locales/en-US/browser/profile/default-bookmarks.ftl
+        - toolkit/locales/en-US/toolkit/about/aboutMozilla.ftl