Adding upstream version 86.0.1.upstream/86.0.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 310 insertions, 0 deletions

diff --git a/tools/lint/perfdocs/verifier.py b/tools/lint/perfdocs/verifier.py
new file mode 100644
index 0000000000..a9a34478eb
--- /dev/null
+++ b/tools/lint/perfdocs/verifier.py
@@ -0,0 +1,310 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import
+
+import jsonschema
+import os
+import re
+
+from perfdocs.logger import PerfDocLogger
+from perfdocs.utils import read_file, read_yaml
+from perfdocs.gatherer import Gatherer
+
+logger = PerfDocLogger()
+
+"""
+Schema for the config.yml file.
+Expecting a YAML file with a format such as this:
+
+name: raptor
+manifest: testing/raptor/raptor/raptor.ini
+static-only: False
+suites:
+    desktop:
+        description: "Desktop tests."
+        tests:
+            raptor-tp6: "Raptor TP6 tests."
+    mobile:
+        description: "Mobile tests"
+    benchmarks:
+        description: "Benchmark tests."
+        tests:
+            wasm: "All wasm tests."
+
+"""
+CONFIG_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "name": {"type": "string"},
+        "manifest": {"type": "string"},
+        "static-only": {"type": "boolean"},
+        "suites": {
+            "type": "object",
+            "properties": {
+                "suite_name": {
+                    "type": "object",
+                    "properties": {
+                        "tests": {
+                            "type": "object",
+                            "properties": {"test_name": {"type": "string"}},
+                        },
+                        "description": {"type": "string"},
+                    },
+                    "required": ["description"],
+                }
+            },
+        },
+    },
+    "required": ["name", "manifest", "static-only", "suites"],
+}
+
+
+class Verifier(object):
+    """
+    Verifier is used for validating the perfdocs folders/tree. In the future,
+    the generator will make use of this class to obtain a validated set of
+    descriptions that can be used to build up a document.
+    """
+
+    def __init__(self, workspace_dir):
+        """
+        Initialize the Verifier.
+
+        :param str workspace_dir: Path to the top-level checkout directory.
+        """
+        self.workspace_dir = workspace_dir
+        self._gatherer = Gatherer(workspace_dir)
+
+    def validate_descriptions(self, framework_info):
+        """
+        Cross-validate the tests found in the manifests and the YAML
+        test definitions. This function doesn't return a valid flag. Instead,
+        the StructDocLogger.VALIDATION_LOG is used to determine validity.
+
+        The validation proceeds as follows:
+            1. Check that all tests/suites in the YAML exist in the manifests.
+                - At the same time, build a list of global descriptions which
+                   define descriptions for groupings of tests.
+            2. Check that all tests/suites found in the manifests exist in the YAML.
+                - For missing tests, check if a global description for them exists.
+
+        As the validation is completed, errors are output into the validation log
+        for any issues that are found.
+
+        :param dict framework_info: Contains information about the framework. See
+            `Gatherer.get_test_list` for information about its structure.
+        """
+        yaml_content = framework_info["yml_content"]
+
+        # Check for any bad test/suite names in the yaml config file
+        global_descriptions = {}
+        for suite, ytests in yaml_content["suites"].items():
+            # Find the suite, then check against the tests within it
+            if framework_info["test_list"].get(suite):
+                global_descriptions[suite] = []
+                if not ytests.get("tests"):
+                    # It's possible a suite entry has no tests
+                    continue
+
+                # Suite found - now check if any tests in YAML
+                # definitions doesn't exist
+                ytests = ytests["tests"]
+                for test_name in ytests:
+                    foundtest = False
+                    for t in framework_info["test_list"][suite]:
+                        tb = os.path.basename(t)
+                        tb = re.sub("\..*", "", tb)
+                        if test_name == tb:
+                            # Found an exact match for the test_name
+                            foundtest = True
+                            break
+                        if test_name in tb:
+                            # Found a 'fuzzy' match for the test_name
+                            # i.e. 'wasm' could exist for all raptor wasm tests
+                            global_descriptions[suite].append(test_name)
+                            foundtest = True
+                            break
+                    if not foundtest:
+                        logger.warning(
+                            "Could not find an existing test for {} - bad test name?".format(
+                                test_name
+                            ),
+                            framework_info["yml_path"],
+                        )
+            else:
+                logger.warning(
+                    "Could not find an existing suite for {} - bad suite name?".format(
+                        suite
+                    ),
+                    framework_info["yml_path"],
+                )
+
+        # Check for any missing tests/suites
+        for suite, test_list in framework_info["test_list"].items():
+            if not yaml_content["suites"].get(suite):
+                # Description doesn't exist for the suite
+                logger.warning(
+                    "Missing suite description for {}".format(suite),
+                    yaml_content["manifest"],
+                )
+                continue
+
+            # If only a description is provided for the suite, assume
+            # that this is a suite-wide description and don't check for
+            # it's tests
+            stests = yaml_content["suites"][suite].get("tests", None)
+            if not stests:
+                continue
+
+            tests_found = 0
+            missing_tests = []
+            test_to_manifest = {}
+            for test_name, manifest_path in test_list.items():
+                tb = os.path.basename(manifest_path)
+                tb = re.sub("\..*", "", tb)
+                if (
+                    stests.get(tb, None) is not None
+                    or stests.get(test_name, None) is not None
+                ):
+                    # Test description exists, continue with the next test
+                    tests_found += 1
+                    continue
+                test_to_manifest[test_name] = manifest_path
+                missing_tests.append(test_name)
+
+            # Check if global test descriptions exist (i.e.
+            # ones that cover all of tp6) for the missing tests
+            new_mtests = []
+            for mt in missing_tests:
+                found = False
+                for test_name in global_descriptions[suite]:
+                    # Global test exists for this missing test
+                    if mt.startswith(test_name):
+                        found = True
+                        break
+                    if test_name in mt:
+                        found = True
+                        break
+                if not found:
+                    new_mtests.append(mt)
+
+            if len(new_mtests):
+                # Output an error for each manifest with a missing
+                # test description
+                for test_name in new_mtests:
+                    logger.warning(
+                        "Could not find a test description for {}".format(test_name),
+                        test_to_manifest[test_name],
+                    )
+
+    def validate_yaml(self, yaml_path):
+        """
+        Validate that the YAML file has all the fields that are
+        required and parse the descriptions into strings in case
+        some are give as relative file paths.
+
+        :param str yaml_path: Path to the YAML to validate.
+        :return bool: True/False => Passed/Failed Validation
+        """
+
+        def _get_description(desc):
+            """
+            Recompute the description in case it's a file.
+            """
+            desc_path = os.path.join(self.workspace_dir, desc)
+            if os.path.exists(desc_path) and os.path.isfile(desc_path):
+                with open(desc_path, "r") as f:
+                    desc = f.readlines()
+            return desc
+
+        def _parse_descriptions(content):
+            for suite, sinfo in content.items():
+                desc = sinfo["description"]
+                sinfo["description"] = _get_description(desc)
+
+                # It's possible that the suite has no tests and
+                # only a description. If they exist, then parse them.
+                if "tests" in sinfo:
+                    for test, desc in sinfo["tests"].items():
+                        sinfo["tests"][test] = _get_description(desc)
+
+        valid = False
+        yaml_content = read_yaml(yaml_path)
+
+        try:
+            jsonschema.validate(instance=yaml_content, schema=CONFIG_SCHEMA)
+            _parse_descriptions(yaml_content["suites"])
+            valid = True
+        except Exception as e:
+            logger.warning("YAML ValidationError: {}".format(str(e)), yaml_path)
+
+        return valid
+
+    def validate_rst_content(self, rst_path):
+        """
+        Validate that the index file given has a {documentation} entry
+        so that the documentation can be inserted there.
+
+        :param str rst_path: Path to the RST file.
+        :return bool: True/False => Passed/Failed Validation
+        """
+        rst_content = read_file(rst_path)
+
+        # Check for a {documentation} entry in some line,
+        # if we can't find one, then the validation fails.
+        valid = False
+        docs_match = re.compile(".*{documentation}.*")
+        for line in rst_content:
+            if docs_match.search(line):
+                valid = True
+                break
+        if not valid:
+            logger.warning(
+                "Cannot find a '{documentation}' entry in the given index file",
+                rst_path,
+            )
+
+        return valid
+
+    def _check_framework_descriptions(self, item):
+        """
+        Helper method for validating descriptions
+        """
+        framework_info = self._gatherer.get_test_list(item)
+        self.validate_descriptions(framework_info)
+
+    def validate_tree(self):
+        """
+        Validate the `perfdocs` directory that was found.
+        Returns True if it is good, false otherwise.
+
+        :return bool: True/False => Passed/Failed Validation
+        """
+        found_good = 0
+
+        # For each framework, check their files and validate descriptions
+        for matched in self._gatherer.perfdocs_tree:
+            # Get the paths to the YAML and RST for this framework
+            matched_yml = os.path.join(matched["path"], matched["yml"])
+            matched_rst = os.path.join(matched["path"], matched["rst"])
+
+            _valid_files = {
+                "yml": self.validate_yaml(matched_yml),
+                "rst": True,
+            }
+            if not read_yaml(matched_yml)["static-only"]:
+                _valid_files["rst"] = self.validate_rst_content(matched_rst)
+
+            # Log independently the errors found for the matched files
+            for file_format, valid in _valid_files.items():
+                if not valid:
+                    logger.log("File validation error: {}".format(file_format))
+            if not all(_valid_files.values()):
+                continue
+            found_good += 1
+
+            self._check_framework_descriptions(matched)
+
+        if not found_good:
+            raise Exception("No valid perfdocs directories found")