# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. import io import os import re from .ini import combine_fields __all__ = ["read_toml", "alphabetize_toml_str", "add_skip_if", "sort_paths"] FILENAME_REGEX = r"^([A-Za-z0-9_./-]*)([Bb][Uu][Gg])([-_]*)([0-9]+)([A-Za-z0-9_./-]*)$" DEFAULT_SECTION = "DEFAULT" def sort_paths_keyfn(k): sort_paths_keyfn.rx = getattr(sort_paths_keyfn, "rx", None) # static if sort_paths_keyfn.rx is None: sort_paths_keyfn.rx = re.compile(FILENAME_REGEX) name = str(k) if name == DEFAULT_SECTION: return "" m = sort_paths_keyfn.rx.findall(name) if len(m) == 1 and len(m[0]) == 5: prefix = m[0][0] # text before "Bug" bug = m[0][1] # the word "Bug" underbar = m[0][2] # underbar or dash (optional) num = m[0][3] # the bug id suffix = m[0][4] # text after the bug id name = f"{prefix}{bug.lower()}{underbar}{int(num):09d}{suffix}" return name return name def sort_paths(paths): """ Returns a list of paths (tests) in a manifest in alphabetical order. Ensures DEFAULT is first and filenames with a bug number are in the proper order. """ return sorted(paths, key=sort_paths_keyfn) def parse_toml_str(contents): """ Parse TOML contents using toml """ import toml error = None manifest = None try: manifest = toml.loads(contents) except toml.TomlDecodeError as pe: error = str(pe) return error, manifest def parse_tomlkit_str(contents): """ Parse TOML contents using tomlkit """ import tomlkit from tomlkit.exceptions import TOMLKitError error = None manifest = None try: manifest = tomlkit.parse(contents) except TOMLKitError as pe: error = str(pe) return error, manifest def read_toml( fp, defaults=None, default=DEFAULT_SECTION, _comments=None, _separators=None, strict=True, handle_defaults=True, document=False, add_line_no=False, ): """ read a .toml file and return a list of [(section, values)] - fp : file pointer or path to read - defaults : default set of variables - default : name of the section for the default section - comments : characters that if they start a line denote a comment - separators : strings that denote key, value separation in order - strict : whether to be strict about parsing - handle_defaults : whether to incorporate defaults into each section - document: read TOML with tomlkit and return source in test["document"] - add_line_no: add the line number where the test name appears in the file to the source. Also, the document variable must be set to True for this flag to work. (This is used only to generate the documentation) """ # variables defaults = defaults or {} default_section = {} sections = [] if isinstance(fp, str): filename = fp fp = io.open(fp, encoding="utf-8") elif hasattr(fp, "name"): filename = fp.name else: filename = "unknown" contents = fp.read() inline_comment_rx = re.compile(r"\s#.*$") if document: # Use tomlkit to parse the file contents error, manifest = parse_tomlkit_str(contents) else: error, manifest = parse_toml_str(contents) if error: raise IOError(f"Error parsing TOML manifest file {filename}: {error}") # handle each section of the manifest for section in manifest.keys(): current_section = {} for key in manifest[section].keys(): val = manifest[section][key] if isinstance(val, bool): # must coerce to lowercase string if val: val = "true" else: val = "false" elif isinstance(val, list): new_vals = "" for v in val: if len(new_vals) > 0: new_vals += os.linesep new_val = str(v).strip() # coerce to str comment_found = inline_comment_rx.search(new_val) if comment_found: new_val = new_val[0 : comment_found.span()[0]] if " = " in new_val: raise Exception( f"Should not assign in {key} condition for {section}" ) new_vals += new_val val = new_vals else: val = str(val).strip() # coerce to str comment_found = inline_comment_rx.search(val) if comment_found: val = val[0 : comment_found.span()[0]] if " = " in val: raise Exception( f"Should not assign in {key} condition for {section}" ) current_section[key] = val if section.lower() == default.lower(): default_section = current_section # DEFAULT does NOT appear in the output else: sections.append((section, current_section)) # merge global defaults with the DEFAULT section defaults = combine_fields(defaults, default_section) if handle_defaults: # merge combined defaults into each section sections = [(i, combine_fields(defaults, j)) for i, j in sections] if document and add_line_no: # Take the line where the test name appears in the file. for i, _ in enumerate(sections): line = contents.split(sections[i][0])[0].count(os.linesep) + 1 manifest.setdefault(sections[i][0], {})["lineno"] = str(line) elif not document: manifest = None return sections, defaults, manifest def alphabetize_toml_str(manifest): """ Will take a TOMLkit manifest document (i.e. from a previous invocation of read_toml(..., document=True) and accessing the document from mp.source_documents[filename]) and return it as a string in sorted order by section (i.e. test file name, taking bug ids into consideration). """ from tomlkit import document, dumps, table from tomlkit.items import Table preamble = "" new_manifest = document() first_section = False sections = {} for k, v in manifest.body: if k is None: preamble += v.as_string() continue if not isinstance(v, Table): raise Exception(f"MP TOML illegal keyval in preamble: {k} = {v}") section = None if not first_section: if k == DEFAULT_SECTION: new_manifest.add(k, v) else: new_manifest.add(DEFAULT_SECTION, table()) first_section = True else: values = v.items() if len(values) == 1: for kk, vv in values: if isinstance(vv, Table): # unquoted, dotted key section = f"{k}.{kk}" sections[section] = vv if section is None: section = str(k).strip("'\"") sections[section] = v if not first_section: new_manifest.add(DEFAULT_SECTION, table()) for section in sort_paths([k for k in sections.keys() if k != DEFAULT_SECTION]): new_manifest.add(section, sections[section]) manifest_str = dumps(new_manifest) # tomlkit fixups manifest_str = preamble + manifest_str.replace('"",]', "]") while manifest_str.endswith("\n\n"): manifest_str = manifest_str[:-1] return manifest_str def _simplify_comment(comment): """Remove any leading #, but preserve leading whitespace in comment""" length = len(comment) i = 0 j = -1 # remove exactly one space while i < length and comment[i] in " #": i += 1 if comment[i] == " ": j += 1 comment = comment[i:] if j > 0: comment = " " * j + comment return comment.rstrip() def add_skip_if(manifest, filename, condition, bug=None): """ Will take a TOMLkit manifest document (i.e. from a previous invocation of read_toml(..., document=True) and accessing the document from mp.source_documents[filename]) and return it as a string in sorted order by section (i.e. test file name, taking bug ids into consideration). """ from tomlkit import array from tomlkit.items import Comment, String, Whitespace if filename not in manifest: raise Exception(f"TOML manifest does not contain section: {filename}") keyvals = manifest[filename] first = None first_comment = "" skip_if = None existing = False # this condition is already present if "skip-if" in keyvals: skip_if = keyvals["skip-if"] if len(skip_if) == 1: for e in skip_if._iter_items(): if not first: if not isinstance(e, Whitespace): first = e.as_string().strip('"') else: c = e.as_string() if c != ",": first_comment += c if skip_if.trivia is not None: first_comment += skip_if.trivia.comment mp_array = array() if skip_if is None: # add the first one line entry to the table mp_array.add_line(condition, indent="", add_comma=False, newline=False) if bug is not None: mp_array.comment(bug) skip_if = {"skip-if": mp_array} keyvals.update(skip_if) else: if first is not None: if first == condition: existing = True if first_comment is not None: mp_array.add_line( first, indent=" ", comment=_simplify_comment(first_comment) ) else: mp_array.add_line(first, indent=" ") if len(skip_if) > 1: e_condition = None e_comment = None for e in skip_if._iter_items(): if isinstance(e, String): if e_condition is not None: if e_comment is not None: mp_array.add_line( e_condition, indent=" ", comment=e_comment ) e_comment = None else: mp_array.add_line(e_condition, indent=" ") e_condition = None if len(e) > 0: e_condition = e.as_string().strip('"') if e_condition == condition: existing = True elif isinstance(e, Comment): e_comment = _simplify_comment(e.as_string()) if e_condition is not None: if e_comment is not None: mp_array.add_line(e_condition, indent=" ", comment=e_comment) else: mp_array.add_line(e_condition, indent=" ") if not existing: if bug is not None: mp_array.add_line(condition, indent=" ", comment=bug) else: mp_array.add_line(condition, indent=" ") mp_array.add_line("", indent="") # fixed in write_toml_str skip_if = {"skip-if": mp_array} del keyvals["skip-if"] keyvals.update(skip_if)