# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import import jsonschema import os import re from perfdocs.logger import PerfDocLogger from perfdocs.utils import read_file, read_yaml from perfdocs.gatherer import Gatherer logger = PerfDocLogger() """ Schema for the config.yml file. Expecting a YAML file with a format such as this: name: raptor manifest: testing/raptor/raptor/raptor.ini static-only: False suites: desktop: description: "Desktop tests." tests: raptor-tp6: "Raptor TP6 tests." mobile: description: "Mobile tests" benchmarks: description: "Benchmark tests." tests: wasm: "All wasm tests." """ CONFIG_SCHEMA = { "type": "object", "properties": { "name": {"type": "string"}, "manifest": {"type": "string"}, "static-only": {"type": "boolean"}, "suites": { "type": "object", "properties": { "suite_name": { "type": "object", "properties": { "tests": { "type": "object", "properties": {"test_name": {"type": "string"}}, }, "description": {"type": "string"}, }, "required": ["description"], } }, }, }, "required": ["name", "manifest", "static-only", "suites"], } class Verifier(object): """ Verifier is used for validating the perfdocs folders/tree. In the future, the generator will make use of this class to obtain a validated set of descriptions that can be used to build up a document. """ def __init__(self, workspace_dir): """ Initialize the Verifier. :param str workspace_dir: Path to the top-level checkout directory. """ self.workspace_dir = workspace_dir self._gatherer = Gatherer(workspace_dir) def validate_descriptions(self, framework_info): """ Cross-validate the tests found in the manifests and the YAML test definitions. This function doesn't return a valid flag. Instead, the StructDocLogger.VALIDATION_LOG is used to determine validity. The validation proceeds as follows: 1. Check that all tests/suites in the YAML exist in the manifests. - At the same time, build a list of global descriptions which define descriptions for groupings of tests. 2. Check that all tests/suites found in the manifests exist in the YAML. - For missing tests, check if a global description for them exists. As the validation is completed, errors are output into the validation log for any issues that are found. :param dict framework_info: Contains information about the framework. See `Gatherer.get_test_list` for information about its structure. """ yaml_content = framework_info["yml_content"] # Check for any bad test/suite names in the yaml config file global_descriptions = {} for suite, ytests in yaml_content["suites"].items(): # Find the suite, then check against the tests within it if framework_info["test_list"].get(suite): global_descriptions[suite] = [] if not ytests.get("tests"): # It's possible a suite entry has no tests continue # Suite found - now check if any tests in YAML # definitions doesn't exist ytests = ytests["tests"] for test_name in ytests: foundtest = False for t in framework_info["test_list"][suite]: tb = os.path.basename(t) tb = re.sub("\..*", "", tb) if test_name == tb: # Found an exact match for the test_name foundtest = True break if test_name in tb: # Found a 'fuzzy' match for the test_name # i.e. 'wasm' could exist for all raptor wasm tests global_descriptions[suite].append(test_name) foundtest = True break if not foundtest: logger.warning( "Could not find an existing test for {} - bad test name?".format( test_name ), framework_info["yml_path"], ) else: logger.warning( "Could not find an existing suite for {} - bad suite name?".format( suite ), framework_info["yml_path"], ) # Check for any missing tests/suites for suite, test_list in framework_info["test_list"].items(): if not yaml_content["suites"].get(suite): # Description doesn't exist for the suite logger.warning( "Missing suite description for {}".format(suite), yaml_content["manifest"], ) continue # If only a description is provided for the suite, assume # that this is a suite-wide description and don't check for # it's tests stests = yaml_content["suites"][suite].get("tests", None) if not stests: continue tests_found = 0 missing_tests = [] test_to_manifest = {} for test_name, manifest_path in test_list.items(): tb = os.path.basename(manifest_path) tb = re.sub("\..*", "", tb) if ( stests.get(tb, None) is not None or stests.get(test_name, None) is not None ): # Test description exists, continue with the next test tests_found += 1 continue test_to_manifest[test_name] = manifest_path missing_tests.append(test_name) # Check if global test descriptions exist (i.e. # ones that cover all of tp6) for the missing tests new_mtests = [] for mt in missing_tests: found = False for test_name in global_descriptions[suite]: # Global test exists for this missing test if mt.startswith(test_name): found = True break if test_name in mt: found = True break if not found: new_mtests.append(mt) if len(new_mtests): # Output an error for each manifest with a missing # test description for test_name in new_mtests: logger.warning( "Could not find a test description for {}".format(test_name), test_to_manifest[test_name], ) def validate_yaml(self, yaml_path): """ Validate that the YAML file has all the fields that are required and parse the descriptions into strings in case some are give as relative file paths. :param str yaml_path: Path to the YAML to validate. :return bool: True/False => Passed/Failed Validation """ def _get_description(desc): """ Recompute the description in case it's a file. """ desc_path = os.path.join(self.workspace_dir, desc) if os.path.exists(desc_path) and os.path.isfile(desc_path): with open(desc_path, "r") as f: desc = f.readlines() return desc def _parse_descriptions(content): for suite, sinfo in content.items(): desc = sinfo["description"] sinfo["description"] = _get_description(desc) # It's possible that the suite has no tests and # only a description. If they exist, then parse them. if "tests" in sinfo: for test, desc in sinfo["tests"].items(): sinfo["tests"][test] = _get_description(desc) valid = False yaml_content = read_yaml(yaml_path) try: jsonschema.validate(instance=yaml_content, schema=CONFIG_SCHEMA) _parse_descriptions(yaml_content["suites"]) valid = True except Exception as e: logger.warning("YAML ValidationError: {}".format(str(e)), yaml_path) return valid def validate_rst_content(self, rst_path): """ Validate that the index file given has a {documentation} entry so that the documentation can be inserted there. :param str rst_path: Path to the RST file. :return bool: True/False => Passed/Failed Validation """ rst_content = read_file(rst_path) # Check for a {documentation} entry in some line, # if we can't find one, then the validation fails. valid = False docs_match = re.compile(".*{documentation}.*") for line in rst_content: if docs_match.search(line): valid = True break if not valid: logger.warning( "Cannot find a '{documentation}' entry in the given index file", rst_path, ) return valid def _check_framework_descriptions(self, item): """ Helper method for validating descriptions """ framework_info = self._gatherer.get_test_list(item) self.validate_descriptions(framework_info) def validate_tree(self): """ Validate the `perfdocs` directory that was found. Returns True if it is good, false otherwise. :return bool: True/False => Passed/Failed Validation """ found_good = 0 # For each framework, check their files and validate descriptions for matched in self._gatherer.perfdocs_tree: # Get the paths to the YAML and RST for this framework matched_yml = os.path.join(matched["path"], matched["yml"]) matched_rst = os.path.join(matched["path"], matched["rst"]) _valid_files = { "yml": self.validate_yaml(matched_yml), "rst": True, } if not read_yaml(matched_yml)["static-only"]: _valid_files["rst"] = self.validate_rst_content(matched_rst) # Log independently the errors found for the matched files for file_format, valid in _valid_files.items(): if not valid: logger.log("File validation error: {}".format(file_format)) if not all(_valid_files.values()): continue found_good += 1 self._check_framework_descriptions(matched) if not found_good: raise Exception("No valid perfdocs directories found")