diff options
Diffstat (limited to 'dom/bindings/mozwebidlcodegen/__init__.py')
| -rw-r--r-- | dom/bindings/mozwebidlcodegen/__init__.py | 681 |
1 files changed, 681 insertions, 0 deletions
diff --git a/dom/bindings/mozwebidlcodegen/__init__.py b/dom/bindings/mozwebidlcodegen/__init__.py new file mode 100644 index 0000000000..8b6e62f345 --- /dev/null +++ b/dom/bindings/mozwebidlcodegen/__init__.py @@ -0,0 +1,681 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This module contains code for managing WebIDL files and bindings for +# the build system. + +import errno +import hashlib +import io +import json +import logging +import os +from copy import deepcopy + +import mozpack.path as mozpath +import six +from mach.mixin.logging import LoggingMixin +from mozbuild.makeutil import Makefile +from mozbuild.pythonutil import iter_modules_in_path +from mozbuild.util import FileAvoidWrite + +# There are various imports in this file in functions to avoid adding +# dependencies to config.status. See bug 949875. + + +class BuildResult(object): + """Represents the result of processing WebIDL files. + + This holds a summary of output file generation during code generation. + """ + + def __init__(self): + # The .webidl files that had their outputs regenerated. + self.inputs = set() + + # The output files that were created. + self.created = set() + + # The output files that changed. + self.updated = set() + + # The output files that didn't change. + self.unchanged = set() + + +class WebIDLCodegenManagerState(dict): + """Holds state for the WebIDL code generation manager. + + State is currently just an extended dict. The internal implementation of + state should be considered a black box to everyone except + WebIDLCodegenManager. But we'll still document it. + + Fields: + + version + The integer version of the format. This is to detect incompatible + changes between state. It should be bumped whenever the format + changes or semantics change. + + webidls + A dictionary holding information about every known WebIDL input. + Keys are the basenames of input WebIDL files. Values are dicts of + metadata. Keys in those dicts are: + + * filename - The full path to the input filename. + * inputs - A set of full paths to other webidl files this webidl + depends on. + * outputs - Set of full output paths that are created/derived from + this file. + * sha1 - The hexidecimal SHA-1 of the input filename from the last + processing time. + + global_inputs + A dictionary defining files that influence all processing. Keys + are full filenames. Values are hexidecimal SHA-1 from the last + processing time. + + dictionaries_convertible_to_js + A set of names of dictionaries that are convertible to JS. + + dictionaries_convertible_from_js + A set of names of dictionaries that are convertible from JS. + """ + + VERSION = 3 + + def __init__(self, fh=None): + self["version"] = self.VERSION + self["webidls"] = {} + self["global_depends"] = {} + + if not fh: + return + + state = json.load(fh) + if state["version"] != self.VERSION: + raise Exception("Unknown state version: %s" % state["version"]) + + self["version"] = state["version"] + self["global_depends"] = state["global_depends"] + + for k, v in state["webidls"].items(): + self["webidls"][k] = v + + # Sets are converted to lists for serialization because JSON + # doesn't support sets. + self["webidls"][k]["inputs"] = set(v["inputs"]) + self["webidls"][k]["outputs"] = set(v["outputs"]) + + self["dictionaries_convertible_to_js"] = set( + state["dictionaries_convertible_to_js"] + ) + + self["dictionaries_convertible_from_js"] = set( + state["dictionaries_convertible_from_js"] + ) + + def dump(self, fh): + """Dump serialized state to a file handle.""" + normalized = deepcopy(self) + + for k, v in self["webidls"].items(): + # Convert sets to lists because JSON doesn't support sets. + normalized["webidls"][k]["outputs"] = sorted(v["outputs"]) + normalized["webidls"][k]["inputs"] = sorted(v["inputs"]) + + normalized["dictionaries_convertible_to_js"] = sorted( + self["dictionaries_convertible_to_js"] + ) + + normalized["dictionaries_convertible_from_js"] = sorted( + self["dictionaries_convertible_from_js"] + ) + + json.dump(normalized, fh, sort_keys=True) + + +class WebIDLCodegenManager(LoggingMixin): + """Manages all code generation around WebIDL. + + To facilitate testing, this object is meant to be generic and reusable. + Paths, etc should be parameters and not hardcoded. + """ + + # Global parser derived declaration files. + GLOBAL_DECLARE_FILES = { + "BindingNames.h", + "GeneratedAtomList.h", + "GeneratedEventList.h", + "PrototypeList.h", + "RegisterBindings.h", + "RegisterShadowRealmBindings.h", + "RegisterWorkerBindings.h", + "RegisterWorkerDebuggerBindings.h", + "RegisterWorkletBindings.h", + "UnionTypes.h", + "WebIDLPrefs.h", + "WebIDLSerializable.h", + } + + # Global parser derived definition files. + GLOBAL_DEFINE_FILES = { + "BindingNames.cpp", + "RegisterBindings.cpp", + "RegisterShadowRealmBindings.cpp", + "RegisterWorkerBindings.cpp", + "RegisterWorkerDebuggerBindings.cpp", + "RegisterWorkletBindings.cpp", + "UnionTypes.cpp", + "PrototypeList.cpp", + "WebIDLPrefs.cpp", + "WebIDLSerializable.cpp", + } + + def __init__( + self, + config_path, + webidl_root, + inputs, + exported_header_dir, + codegen_dir, + state_path, + cache_dir=None, + make_deps_path=None, + make_deps_target=None, + ): + """Create an instance that manages WebIDLs in the build system. + + config_path refers to a WebIDL config file (e.g. Bindings.conf). + inputs is a 4-tuple describing the input .webidl files and how to + process them. Members are: + (set(.webidl files), set(basenames of exported files), + set(basenames of generated events files), + set(example interface names)) + + exported_header_dir and codegen_dir are directories where generated + files will be written to. + state_path is the path to a file that will receive JSON state from our + actions. + make_deps_path is the path to a make dependency file that we can + optionally write. + make_deps_target is the target that receives the make dependencies. It + must be defined if using make_deps_path. + """ + self.populate_logger() + + input_paths, exported_stems, generated_events_stems, example_interfaces = inputs + + self._config_path = config_path + self._webidl_root = webidl_root + self._input_paths = set(input_paths) + self._exported_stems = set(exported_stems) + self._generated_events_stems = set(generated_events_stems) + self._generated_events_stems_as_array = generated_events_stems + self._example_interfaces = set(example_interfaces) + self._exported_header_dir = exported_header_dir + self._codegen_dir = codegen_dir + self._state_path = state_path + self._cache_dir = cache_dir + self._make_deps_path = make_deps_path + self._make_deps_target = make_deps_target + + if (make_deps_path and not make_deps_target) or ( + not make_deps_path and make_deps_target + ): + raise Exception( + "Must define both make_deps_path and make_deps_target " + "if one is defined." + ) + + self._parser_results = None + self._config = None + self._state = WebIDLCodegenManagerState() + + if os.path.exists(state_path): + with io.open(state_path, "r") as fh: + try: + self._state = WebIDLCodegenManagerState(fh=fh) + except Exception as e: + self.log( + logging.WARN, + "webidl_bad_state", + {"msg": str(e)}, + "Bad WebIDL state: {msg}", + ) + + @property + def config(self): + if not self._config: + self._parse_webidl() + + return self._config + + def generate_build_files(self): + """Generate files required for the build. + + This function is in charge of generating all the .h/.cpp files derived + from input .webidl files. Please note that there are build actions + required to produce .webidl files and these build actions are + explicitly not captured here: this function assumes all .webidl files + are present and up to date. + + This routine is called as part of the build to ensure files that need + to exist are present and up to date. This routine may not be called if + the build dependencies (generated as a result of calling this the first + time) say everything is up to date. + + Because reprocessing outputs for every .webidl on every invocation + is expensive, we only regenerate the minimal set of files on every + invocation. The rules for deciding what needs done are roughly as + follows: + + 1. If any .webidl changes, reparse all .webidl files and regenerate + the global derived files. Only regenerate output files (.h/.cpp) + impacted by the modified .webidl files. + 2. If an non-.webidl dependency (Python files, config file) changes, + assume everything is out of date and regenerate the world. This + is because changes in those could globally impact every output + file. + 3. If an output file is missing, ensure it is present by performing + necessary regeneration. + """ + # Despite #1 above, we assume the build system is smart enough to not + # invoke us if nothing has changed. Therefore, any invocation means + # something has changed. And, if anything has changed, we need to + # parse the WebIDL. + self._parse_webidl() + + result = BuildResult() + + # If we parse, we always update globals - they are cheap and it is + # easier that way. + created, updated, unchanged = self._write_global_derived() + result.created |= created + result.updated |= updated + result.unchanged |= unchanged + + # If any of the extra dependencies changed, regenerate the world. + global_changed, global_hashes = self._global_dependencies_changed() + if global_changed: + # Make a copy because we may modify. + changed_inputs = set(self._input_paths) + else: + changed_inputs = self._compute_changed_inputs() + + self._state["global_depends"] = global_hashes + self._state["dictionaries_convertible_to_js"] = set( + d.identifier.name for d in self._config.getDictionariesConvertibleToJS() + ) + self._state["dictionaries_convertible_from_js"] = set( + d.identifier.name for d in self._config.getDictionariesConvertibleFromJS() + ) + + # Generate bindings from .webidl files. + for filename in sorted(changed_inputs): + basename = mozpath.basename(filename) + result.inputs.add(filename) + written, deps = self._generate_build_files_for_webidl(filename) + result.created |= written[0] + result.updated |= written[1] + result.unchanged |= written[2] + + self._state["webidls"][basename] = dict( + filename=filename, + outputs=written[0] | written[1] | written[2], + inputs=set(deps), + sha1=self._input_hashes[filename], + ) + + # Process some special interfaces required for testing. + for interface in self._example_interfaces: + written = self.generate_example_files(interface) + result.created |= written[0] + result.updated |= written[1] + result.unchanged |= written[2] + + # Generate a make dependency file. + if self._make_deps_path: + mk = Makefile() + codegen_rule = mk.create_rule([self._make_deps_target]) + codegen_rule.add_dependencies( + six.ensure_text(s) for s in global_hashes.keys() + ) + codegen_rule.add_dependencies(six.ensure_text(p) for p in self._input_paths) + + with FileAvoidWrite(self._make_deps_path) as fh: + mk.dump(fh) + + self._save_state() + + return result + + def generate_example_files(self, interface): + """Generates example files for a given interface.""" + from Codegen import CGExampleRoot + + root = CGExampleRoot(self.config, interface) + + example_paths = self._example_paths(interface) + for path in example_paths: + print("Generating {}".format(path)) + + return self._maybe_write_codegen(root, *example_paths) + + def _parse_webidl(self): + import WebIDL + from Configuration import Configuration + + self.log( + logging.INFO, + "webidl_parse", + {"count": len(self._input_paths)}, + "Parsing {count} WebIDL files.", + ) + + hashes = {} + parser = WebIDL.Parser(self._cache_dir, lexer=None) + + for path in sorted(self._input_paths): + with io.open(path, "r", encoding="utf-8") as fh: + data = fh.read() + hashes[path] = hashlib.sha1(six.ensure_binary(data)).hexdigest() + parser.parse(data, path) + + # Only these directories may contain WebIDL files with interfaces + # which are exposed to the web. WebIDL files in these roots may not + # be changed without DOM peer review. + # + # Other directories may contain WebIDL files as long as they only + # contain ChromeOnly interfaces. These are not subject to mandatory + # DOM peer review. + web_roots = ( + # The main WebIDL root. + self._webidl_root, + # The binding config root, which contains some test-only + # interfaces. + os.path.dirname(self._config_path), + # The objdir sub-directory which contains generated WebIDL files. + self._codegen_dir, + ) + + self._parser_results = parser.finish() + self._config = Configuration( + self._config_path, + web_roots, + self._parser_results, + self._generated_events_stems_as_array, + ) + self._input_hashes = hashes + + def _write_global_derived(self): + from Codegen import GlobalGenRoots + + things = [("declare", f) for f in self.GLOBAL_DECLARE_FILES] + things.extend(("define", f) for f in self.GLOBAL_DEFINE_FILES) + + result = (set(), set(), set()) + + for what, filename in things: + stem = mozpath.splitext(filename)[0] + root = getattr(GlobalGenRoots, stem)(self._config) + + if what == "declare": + code = root.declare() + output_root = self._exported_header_dir + elif what == "define": + code = root.define() + output_root = self._codegen_dir + else: + raise Exception("Unknown global gen type: %s" % what) + + output_path = mozpath.join(output_root, filename) + self._maybe_write_file(output_path, code, result) + + return result + + def _compute_changed_inputs(self): + """Compute the set of input files that need to be regenerated.""" + changed_inputs = set() + expected_outputs = self.expected_build_output_files() + + # Look for missing output files. + if any(not os.path.exists(f) for f in expected_outputs): + # FUTURE Bug 940469 Only regenerate minimum set. + changed_inputs |= self._input_paths + + # That's it for examining output files. We /could/ examine SHA-1's of + # output files from a previous run to detect modifications. But that's + # a lot of extra work and most build systems don't do that anyway. + + # Now we move on to the input files. + old_hashes = {v["filename"]: v["sha1"] for v in self._state["webidls"].values()} + + old_filenames = set(old_hashes.keys()) + new_filenames = self._input_paths + + # If an old file has disappeared or a new file has arrived, mark + # it. + changed_inputs |= old_filenames ^ new_filenames + + # For the files in common between runs, compare content. If the file + # has changed, mark it. We don't need to perform mtime comparisons + # because content is a stronger validator. + for filename in old_filenames & new_filenames: + if old_hashes[filename] != self._input_hashes[filename]: + changed_inputs.add(filename) + + # We've now populated the base set of inputs that have changed. + + # Inherit dependencies from previous run. The full set of dependencies + # is associated with each record, so we don't need to perform any fancy + # graph traversal. + for v in self._state["webidls"].values(): + if any(dep for dep in v["inputs"] if dep in changed_inputs): + changed_inputs.add(v["filename"]) + + # Now check for changes to the set of dictionaries that are convertible to JS + oldDictionariesConvertibleToJS = self._state["dictionaries_convertible_to_js"] + newDictionariesConvertibleToJS = self._config.getDictionariesConvertibleToJS() + newNames = set(d.identifier.name for d in newDictionariesConvertibleToJS) + changedDictionaryNames = oldDictionariesConvertibleToJS ^ newNames + + # Now check for changes to the set of dictionaries that are convertible from JS + oldDictionariesConvertibleFromJS = self._state[ + "dictionaries_convertible_from_js" + ] + newDictionariesConvertibleFromJS = ( + self._config.getDictionariesConvertibleFromJS() + ) + newNames = set(d.identifier.name for d in newDictionariesConvertibleFromJS) + changedDictionaryNames |= oldDictionariesConvertibleFromJS ^ newNames + + for name in changedDictionaryNames: + d = self._config.getDictionaryIfExists(name) + if d: + changed_inputs.add(d.filename()) + + # Only use paths that are known to our current state. + # This filters out files that were deleted or changed type (e.g. from + # static to preprocessed). + return changed_inputs & self._input_paths + + def _binding_info(self, p): + """Compute binding metadata for an input path. + + Returns a tuple of: + + (stem, binding_stem, is_event, output_files) + + output_files is itself a tuple. The first two items are the binding + header and C++ paths, respectively. The 2nd pair are the event header + and C++ paths or None if this isn't an event binding. + """ + basename = mozpath.basename(p) + stem = mozpath.splitext(basename)[0] + binding_stem = "%sBinding" % stem + + if stem in self._exported_stems: + header_dir = self._exported_header_dir + else: + header_dir = self._codegen_dir + + is_event = stem in self._generated_events_stems + + files = ( + mozpath.join(header_dir, "%s.h" % binding_stem), + mozpath.join(self._codegen_dir, "%s.cpp" % binding_stem), + mozpath.join(header_dir, "%s.h" % stem) if is_event else None, + mozpath.join(self._codegen_dir, "%s.cpp" % stem) if is_event else None, + ) + + return stem, binding_stem, is_event, header_dir, files + + def _example_paths(self, interface): + return ( + mozpath.join(self._codegen_dir, "%s-example.h" % interface), + mozpath.join(self._codegen_dir, "%s-example.cpp" % interface), + ) + + def expected_build_output_files(self): + """Obtain the set of files generate_build_files() should write.""" + paths = set() + + # Account for global generation. + for p in self.GLOBAL_DECLARE_FILES: + paths.add(mozpath.join(self._exported_header_dir, p)) + for p in self.GLOBAL_DEFINE_FILES: + paths.add(mozpath.join(self._codegen_dir, p)) + + for p in self._input_paths: + stem, binding_stem, is_event, header_dir, files = self._binding_info(p) + paths |= {f for f in files if f} + + for interface in self._example_interfaces: + for p in self._example_paths(interface): + paths.add(p) + + return paths + + def _generate_build_files_for_webidl(self, filename): + from Codegen import CGBindingRoot, CGEventRoot + + self.log( + logging.INFO, + "webidl_generate_build_for_input", + {"filename": filename}, + "Generating WebIDL files derived from {filename}", + ) + + stem, binding_stem, is_event, header_dir, files = self._binding_info(filename) + root = CGBindingRoot(self._config, binding_stem, filename) + + result = self._maybe_write_codegen(root, files[0], files[1]) + + if is_event: + generated_event = CGEventRoot(self._config, stem) + result = self._maybe_write_codegen( + generated_event, files[2], files[3], result + ) + + return result, root.deps() + + def _global_dependencies_changed(self): + """Determine whether the global dependencies have changed.""" + current_files = set(iter_modules_in_path(mozpath.dirname(__file__))) + + # We need to catch other .py files from /dom/bindings. We assume these + # are in the same directory as the config file. + current_files |= set(iter_modules_in_path(mozpath.dirname(self._config_path))) + + current_files.add(self._config_path) + + current_hashes = {} + for f in current_files: + # This will fail if the file doesn't exist. If a current global + # dependency doesn't exist, something else is wrong. + with io.open(f, "rb") as fh: + current_hashes[f] = hashlib.sha1(fh.read()).hexdigest() + + # The set of files has changed. + if current_files ^ set(self._state["global_depends"].keys()): + return True, current_hashes + + # Compare hashes. + for f, sha1 in current_hashes.items(): + if sha1 != self._state["global_depends"][f]: + return True, current_hashes + + return False, current_hashes + + def _save_state(self): + with io.open(self._state_path, "w", newline="\n") as fh: + self._state.dump(fh) + + def _maybe_write_codegen(self, obj, declare_path, define_path, result=None): + assert declare_path and define_path + if not result: + result = (set(), set(), set()) + + self._maybe_write_file(declare_path, obj.declare(), result) + self._maybe_write_file(define_path, obj.define(), result) + + return result + + def _maybe_write_file(self, path, content, result): + fh = FileAvoidWrite(path) + fh.write(content) + existed, updated = fh.close() + + if not existed: + result[0].add(path) + elif updated: + result[1].add(path) + else: + result[2].add(path) + + +def create_build_system_manager(topsrcdir=None, topobjdir=None, dist_dir=None): + """Create a WebIDLCodegenManager for use by the build system.""" + if topsrcdir is None: + assert topobjdir is None and dist_dir is None + import buildconfig + + topsrcdir = buildconfig.topsrcdir + topobjdir = buildconfig.topobjdir + dist_dir = buildconfig.substs["DIST"] + + src_dir = os.path.join(topsrcdir, "dom", "bindings") + obj_dir = os.path.join(topobjdir, "dom", "bindings") + webidl_root = os.path.join(topsrcdir, "dom", "webidl") + + with io.open(os.path.join(obj_dir, "file-lists.json"), "r") as fh: + files = json.load(fh) + + inputs = ( + files["webidls"], + files["exported_stems"], + files["generated_events_stems"], + files["example_interfaces"], + ) + + cache_dir = os.path.join(obj_dir, "_cache") + try: + os.makedirs(cache_dir) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + return WebIDLCodegenManager( + os.path.join(src_dir, "Bindings.conf"), + webidl_root, + inputs, + os.path.join(dist_dir, "include", "mozilla", "dom"), + obj_dir, + os.path.join(obj_dir, "codegen.json"), + cache_dir=cache_dir, + # The make rules include a codegen.pp file containing dependencies. + make_deps_path=os.path.join(obj_dir, "codegen.pp"), + make_deps_target="webidl.stub", + ) |