diff options
Diffstat (limited to 'dom/bindings/mozwebidlcodegen')
| -rw-r--r-- | dom/bindings/mozwebidlcodegen/__init__.py | 681 | ||||
| -rw-r--r-- | dom/bindings/mozwebidlcodegen/test/Child.webidl | 3 | ||||
| -rw-r--r-- | dom/bindings/mozwebidlcodegen/test/ExampleBinding.webidl | 3 | ||||
| -rw-r--r-- | dom/bindings/mozwebidlcodegen/test/Parent.webidl | 3 | ||||
| -rw-r--r-- | dom/bindings/mozwebidlcodegen/test/TestEvent.webidl | 13 | ||||
| -rw-r--r-- | dom/bindings/mozwebidlcodegen/test/python.ini | 4 | ||||
| -rw-r--r-- | dom/bindings/mozwebidlcodegen/test/test_mozwebidlcodegen.py | 297 |
7 files changed, 1004 insertions, 0 deletions
diff --git a/dom/bindings/mozwebidlcodegen/__init__.py b/dom/bindings/mozwebidlcodegen/__init__.py new file mode 100644 index 0000000000..8b6e62f345 --- /dev/null +++ b/dom/bindings/mozwebidlcodegen/__init__.py @@ -0,0 +1,681 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This module contains code for managing WebIDL files and bindings for +# the build system. + +import errno +import hashlib +import io +import json +import logging +import os +from copy import deepcopy + +import mozpack.path as mozpath +import six +from mach.mixin.logging import LoggingMixin +from mozbuild.makeutil import Makefile +from mozbuild.pythonutil import iter_modules_in_path +from mozbuild.util import FileAvoidWrite + +# There are various imports in this file in functions to avoid adding +# dependencies to config.status. See bug 949875. + + +class BuildResult(object): + """Represents the result of processing WebIDL files. + + This holds a summary of output file generation during code generation. + """ + + def __init__(self): + # The .webidl files that had their outputs regenerated. + self.inputs = set() + + # The output files that were created. + self.created = set() + + # The output files that changed. + self.updated = set() + + # The output files that didn't change. + self.unchanged = set() + + +class WebIDLCodegenManagerState(dict): + """Holds state for the WebIDL code generation manager. + + State is currently just an extended dict. The internal implementation of + state should be considered a black box to everyone except + WebIDLCodegenManager. But we'll still document it. + + Fields: + + version + The integer version of the format. This is to detect incompatible + changes between state. It should be bumped whenever the format + changes or semantics change. + + webidls + A dictionary holding information about every known WebIDL input. + Keys are the basenames of input WebIDL files. Values are dicts of + metadata. Keys in those dicts are: + + * filename - The full path to the input filename. + * inputs - A set of full paths to other webidl files this webidl + depends on. + * outputs - Set of full output paths that are created/derived from + this file. + * sha1 - The hexidecimal SHA-1 of the input filename from the last + processing time. + + global_inputs + A dictionary defining files that influence all processing. Keys + are full filenames. Values are hexidecimal SHA-1 from the last + processing time. + + dictionaries_convertible_to_js + A set of names of dictionaries that are convertible to JS. + + dictionaries_convertible_from_js + A set of names of dictionaries that are convertible from JS. + """ + + VERSION = 3 + + def __init__(self, fh=None): + self["version"] = self.VERSION + self["webidls"] = {} + self["global_depends"] = {} + + if not fh: + return + + state = json.load(fh) + if state["version"] != self.VERSION: + raise Exception("Unknown state version: %s" % state["version"]) + + self["version"] = state["version"] + self["global_depends"] = state["global_depends"] + + for k, v in state["webidls"].items(): + self["webidls"][k] = v + + # Sets are converted to lists for serialization because JSON + # doesn't support sets. + self["webidls"][k]["inputs"] = set(v["inputs"]) + self["webidls"][k]["outputs"] = set(v["outputs"]) + + self["dictionaries_convertible_to_js"] = set( + state["dictionaries_convertible_to_js"] + ) + + self["dictionaries_convertible_from_js"] = set( + state["dictionaries_convertible_from_js"] + ) + + def dump(self, fh): + """Dump serialized state to a file handle.""" + normalized = deepcopy(self) + + for k, v in self["webidls"].items(): + # Convert sets to lists because JSON doesn't support sets. + normalized["webidls"][k]["outputs"] = sorted(v["outputs"]) + normalized["webidls"][k]["inputs"] = sorted(v["inputs"]) + + normalized["dictionaries_convertible_to_js"] = sorted( + self["dictionaries_convertible_to_js"] + ) + + normalized["dictionaries_convertible_from_js"] = sorted( + self["dictionaries_convertible_from_js"] + ) + + json.dump(normalized, fh, sort_keys=True) + + +class WebIDLCodegenManager(LoggingMixin): + """Manages all code generation around WebIDL. + + To facilitate testing, this object is meant to be generic and reusable. + Paths, etc should be parameters and not hardcoded. + """ + + # Global parser derived declaration files. + GLOBAL_DECLARE_FILES = { + "BindingNames.h", + "GeneratedAtomList.h", + "GeneratedEventList.h", + "PrototypeList.h", + "RegisterBindings.h", + "RegisterShadowRealmBindings.h", + "RegisterWorkerBindings.h", + "RegisterWorkerDebuggerBindings.h", + "RegisterWorkletBindings.h", + "UnionTypes.h", + "WebIDLPrefs.h", + "WebIDLSerializable.h", + } + + # Global parser derived definition files. + GLOBAL_DEFINE_FILES = { + "BindingNames.cpp", + "RegisterBindings.cpp", + "RegisterShadowRealmBindings.cpp", + "RegisterWorkerBindings.cpp", + "RegisterWorkerDebuggerBindings.cpp", + "RegisterWorkletBindings.cpp", + "UnionTypes.cpp", + "PrototypeList.cpp", + "WebIDLPrefs.cpp", + "WebIDLSerializable.cpp", + } + + def __init__( + self, + config_path, + webidl_root, + inputs, + exported_header_dir, + codegen_dir, + state_path, + cache_dir=None, + make_deps_path=None, + make_deps_target=None, + ): + """Create an instance that manages WebIDLs in the build system. + + config_path refers to a WebIDL config file (e.g. Bindings.conf). + inputs is a 4-tuple describing the input .webidl files and how to + process them. Members are: + (set(.webidl files), set(basenames of exported files), + set(basenames of generated events files), + set(example interface names)) + + exported_header_dir and codegen_dir are directories where generated + files will be written to. + state_path is the path to a file that will receive JSON state from our + actions. + make_deps_path is the path to a make dependency file that we can + optionally write. + make_deps_target is the target that receives the make dependencies. It + must be defined if using make_deps_path. + """ + self.populate_logger() + + input_paths, exported_stems, generated_events_stems, example_interfaces = inputs + + self._config_path = config_path + self._webidl_root = webidl_root + self._input_paths = set(input_paths) + self._exported_stems = set(exported_stems) + self._generated_events_stems = set(generated_events_stems) + self._generated_events_stems_as_array = generated_events_stems + self._example_interfaces = set(example_interfaces) + self._exported_header_dir = exported_header_dir + self._codegen_dir = codegen_dir + self._state_path = state_path + self._cache_dir = cache_dir + self._make_deps_path = make_deps_path + self._make_deps_target = make_deps_target + + if (make_deps_path and not make_deps_target) or ( + not make_deps_path and make_deps_target + ): + raise Exception( + "Must define both make_deps_path and make_deps_target " + "if one is defined." + ) + + self._parser_results = None + self._config = None + self._state = WebIDLCodegenManagerState() + + if os.path.exists(state_path): + with io.open(state_path, "r") as fh: + try: + self._state = WebIDLCodegenManagerState(fh=fh) + except Exception as e: + self.log( + logging.WARN, + "webidl_bad_state", + {"msg": str(e)}, + "Bad WebIDL state: {msg}", + ) + + @property + def config(self): + if not self._config: + self._parse_webidl() + + return self._config + + def generate_build_files(self): + """Generate files required for the build. + + This function is in charge of generating all the .h/.cpp files derived + from input .webidl files. Please note that there are build actions + required to produce .webidl files and these build actions are + explicitly not captured here: this function assumes all .webidl files + are present and up to date. + + This routine is called as part of the build to ensure files that need + to exist are present and up to date. This routine may not be called if + the build dependencies (generated as a result of calling this the first + time) say everything is up to date. + + Because reprocessing outputs for every .webidl on every invocation + is expensive, we only regenerate the minimal set of files on every + invocation. The rules for deciding what needs done are roughly as + follows: + + 1. If any .webidl changes, reparse all .webidl files and regenerate + the global derived files. Only regenerate output files (.h/.cpp) + impacted by the modified .webidl files. + 2. If an non-.webidl dependency (Python files, config file) changes, + assume everything is out of date and regenerate the world. This + is because changes in those could globally impact every output + file. + 3. If an output file is missing, ensure it is present by performing + necessary regeneration. + """ + # Despite #1 above, we assume the build system is smart enough to not + # invoke us if nothing has changed. Therefore, any invocation means + # something has changed. And, if anything has changed, we need to + # parse the WebIDL. + self._parse_webidl() + + result = BuildResult() + + # If we parse, we always update globals - they are cheap and it is + # easier that way. + created, updated, unchanged = self._write_global_derived() + result.created |= created + result.updated |= updated + result.unchanged |= unchanged + + # If any of the extra dependencies changed, regenerate the world. + global_changed, global_hashes = self._global_dependencies_changed() + if global_changed: + # Make a copy because we may modify. + changed_inputs = set(self._input_paths) + else: + changed_inputs = self._compute_changed_inputs() + + self._state["global_depends"] = global_hashes + self._state["dictionaries_convertible_to_js"] = set( + d.identifier.name for d in self._config.getDictionariesConvertibleToJS() + ) + self._state["dictionaries_convertible_from_js"] = set( + d.identifier.name for d in self._config.getDictionariesConvertibleFromJS() + ) + + # Generate bindings from .webidl files. + for filename in sorted(changed_inputs): + basename = mozpath.basename(filename) + result.inputs.add(filename) + written, deps = self._generate_build_files_for_webidl(filename) + result.created |= written[0] + result.updated |= written[1] + result.unchanged |= written[2] + + self._state["webidls"][basename] = dict( + filename=filename, + outputs=written[0] | written[1] | written[2], + inputs=set(deps), + sha1=self._input_hashes[filename], + ) + + # Process some special interfaces required for testing. + for interface in self._example_interfaces: + written = self.generate_example_files(interface) + result.created |= written[0] + result.updated |= written[1] + result.unchanged |= written[2] + + # Generate a make dependency file. + if self._make_deps_path: + mk = Makefile() + codegen_rule = mk.create_rule([self._make_deps_target]) + codegen_rule.add_dependencies( + six.ensure_text(s) for s in global_hashes.keys() + ) + codegen_rule.add_dependencies(six.ensure_text(p) for p in self._input_paths) + + with FileAvoidWrite(self._make_deps_path) as fh: + mk.dump(fh) + + self._save_state() + + return result + + def generate_example_files(self, interface): + """Generates example files for a given interface.""" + from Codegen import CGExampleRoot + + root = CGExampleRoot(self.config, interface) + + example_paths = self._example_paths(interface) + for path in example_paths: + print("Generating {}".format(path)) + + return self._maybe_write_codegen(root, *example_paths) + + def _parse_webidl(self): + import WebIDL + from Configuration import Configuration + + self.log( + logging.INFO, + "webidl_parse", + {"count": len(self._input_paths)}, + "Parsing {count} WebIDL files.", + ) + + hashes = {} + parser = WebIDL.Parser(self._cache_dir, lexer=None) + + for path in sorted(self._input_paths): + with io.open(path, "r", encoding="utf-8") as fh: + data = fh.read() + hashes[path] = hashlib.sha1(six.ensure_binary(data)).hexdigest() + parser.parse(data, path) + + # Only these directories may contain WebIDL files with interfaces + # which are exposed to the web. WebIDL files in these roots may not + # be changed without DOM peer review. + # + # Other directories may contain WebIDL files as long as they only + # contain ChromeOnly interfaces. These are not subject to mandatory + # DOM peer review. + web_roots = ( + # The main WebIDL root. + self._webidl_root, + # The binding config root, which contains some test-only + # interfaces. + os.path.dirname(self._config_path), + # The objdir sub-directory which contains generated WebIDL files. + self._codegen_dir, + ) + + self._parser_results = parser.finish() + self._config = Configuration( + self._config_path, + web_roots, + self._parser_results, + self._generated_events_stems_as_array, + ) + self._input_hashes = hashes + + def _write_global_derived(self): + from Codegen import GlobalGenRoots + + things = [("declare", f) for f in self.GLOBAL_DECLARE_FILES] + things.extend(("define", f) for f in self.GLOBAL_DEFINE_FILES) + + result = (set(), set(), set()) + + for what, filename in things: + stem = mozpath.splitext(filename)[0] + root = getattr(GlobalGenRoots, stem)(self._config) + + if what == "declare": + code = root.declare() + output_root = self._exported_header_dir + elif what == "define": + code = root.define() + output_root = self._codegen_dir + else: + raise Exception("Unknown global gen type: %s" % what) + + output_path = mozpath.join(output_root, filename) + self._maybe_write_file(output_path, code, result) + + return result + + def _compute_changed_inputs(self): + """Compute the set of input files that need to be regenerated.""" + changed_inputs = set() + expected_outputs = self.expected_build_output_files() + + # Look for missing output files. + if any(not os.path.exists(f) for f in expected_outputs): + # FUTURE Bug 940469 Only regenerate minimum set. + changed_inputs |= self._input_paths + + # That's it for examining output files. We /could/ examine SHA-1's of + # output files from a previous run to detect modifications. But that's + # a lot of extra work and most build systems don't do that anyway. + + # Now we move on to the input files. + old_hashes = {v["filename"]: v["sha1"] for v in self._state["webidls"].values()} + + old_filenames = set(old_hashes.keys()) + new_filenames = self._input_paths + + # If an old file has disappeared or a new file has arrived, mark + # it. + changed_inputs |= old_filenames ^ new_filenames + + # For the files in common between runs, compare content. If the file + # has changed, mark it. We don't need to perform mtime comparisons + # because content is a stronger validator. + for filename in old_filenames & new_filenames: + if old_hashes[filename] != self._input_hashes[filename]: + changed_inputs.add(filename) + + # We've now populated the base set of inputs that have changed. + + # Inherit dependencies from previous run. The full set of dependencies + # is associated with each record, so we don't need to perform any fancy + # graph traversal. + for v in self._state["webidls"].values(): + if any(dep for dep in v["inputs"] if dep in changed_inputs): + changed_inputs.add(v["filename"]) + + # Now check for changes to the set of dictionaries that are convertible to JS + oldDictionariesConvertibleToJS = self._state["dictionaries_convertible_to_js"] + newDictionariesConvertibleToJS = self._config.getDictionariesConvertibleToJS() + newNames = set(d.identifier.name for d in newDictionariesConvertibleToJS) + changedDictionaryNames = oldDictionariesConvertibleToJS ^ newNames + + # Now check for changes to the set of dictionaries that are convertible from JS + oldDictionariesConvertibleFromJS = self._state[ + "dictionaries_convertible_from_js" + ] + newDictionariesConvertibleFromJS = ( + self._config.getDictionariesConvertibleFromJS() + ) + newNames = set(d.identifier.name for d in newDictionariesConvertibleFromJS) + changedDictionaryNames |= oldDictionariesConvertibleFromJS ^ newNames + + for name in changedDictionaryNames: + d = self._config.getDictionaryIfExists(name) + if d: + changed_inputs.add(d.filename()) + + # Only use paths that are known to our current state. + # This filters out files that were deleted or changed type (e.g. from + # static to preprocessed). + return changed_inputs & self._input_paths + + def _binding_info(self, p): + """Compute binding metadata for an input path. + + Returns a tuple of: + + (stem, binding_stem, is_event, output_files) + + output_files is itself a tuple. The first two items are the binding + header and C++ paths, respectively. The 2nd pair are the event header + and C++ paths or None if this isn't an event binding. + """ + basename = mozpath.basename(p) + stem = mozpath.splitext(basename)[0] + binding_stem = "%sBinding" % stem + + if stem in self._exported_stems: + header_dir = self._exported_header_dir + else: + header_dir = self._codegen_dir + + is_event = stem in self._generated_events_stems + + files = ( + mozpath.join(header_dir, "%s.h" % binding_stem), + mozpath.join(self._codegen_dir, "%s.cpp" % binding_stem), + mozpath.join(header_dir, "%s.h" % stem) if is_event else None, + mozpath.join(self._codegen_dir, "%s.cpp" % stem) if is_event else None, + ) + + return stem, binding_stem, is_event, header_dir, files + + def _example_paths(self, interface): + return ( + mozpath.join(self._codegen_dir, "%s-example.h" % interface), + mozpath.join(self._codegen_dir, "%s-example.cpp" % interface), + ) + + def expected_build_output_files(self): + """Obtain the set of files generate_build_files() should write.""" + paths = set() + + # Account for global generation. + for p in self.GLOBAL_DECLARE_FILES: + paths.add(mozpath.join(self._exported_header_dir, p)) + for p in self.GLOBAL_DEFINE_FILES: + paths.add(mozpath.join(self._codegen_dir, p)) + + for p in self._input_paths: + stem, binding_stem, is_event, header_dir, files = self._binding_info(p) + paths |= {f for f in files if f} + + for interface in self._example_interfaces: + for p in self._example_paths(interface): + paths.add(p) + + return paths + + def _generate_build_files_for_webidl(self, filename): + from Codegen import CGBindingRoot, CGEventRoot + + self.log( + logging.INFO, + "webidl_generate_build_for_input", + {"filename": filename}, + "Generating WebIDL files derived from {filename}", + ) + + stem, binding_stem, is_event, header_dir, files = self._binding_info(filename) + root = CGBindingRoot(self._config, binding_stem, filename) + + result = self._maybe_write_codegen(root, files[0], files[1]) + + if is_event: + generated_event = CGEventRoot(self._config, stem) + result = self._maybe_write_codegen( + generated_event, files[2], files[3], result + ) + + return result, root.deps() + + def _global_dependencies_changed(self): + """Determine whether the global dependencies have changed.""" + current_files = set(iter_modules_in_path(mozpath.dirname(__file__))) + + # We need to catch other .py files from /dom/bindings. We assume these + # are in the same directory as the config file. + current_files |= set(iter_modules_in_path(mozpath.dirname(self._config_path))) + + current_files.add(self._config_path) + + current_hashes = {} + for f in current_files: + # This will fail if the file doesn't exist. If a current global + # dependency doesn't exist, something else is wrong. + with io.open(f, "rb") as fh: + current_hashes[f] = hashlib.sha1(fh.read()).hexdigest() + + # The set of files has changed. + if current_files ^ set(self._state["global_depends"].keys()): + return True, current_hashes + + # Compare hashes. + for f, sha1 in current_hashes.items(): + if sha1 != self._state["global_depends"][f]: + return True, current_hashes + + return False, current_hashes + + def _save_state(self): + with io.open(self._state_path, "w", newline="\n") as fh: + self._state.dump(fh) + + def _maybe_write_codegen(self, obj, declare_path, define_path, result=None): + assert declare_path and define_path + if not result: + result = (set(), set(), set()) + + self._maybe_write_file(declare_path, obj.declare(), result) + self._maybe_write_file(define_path, obj.define(), result) + + return result + + def _maybe_write_file(self, path, content, result): + fh = FileAvoidWrite(path) + fh.write(content) + existed, updated = fh.close() + + if not existed: + result[0].add(path) + elif updated: + result[1].add(path) + else: + result[2].add(path) + + +def create_build_system_manager(topsrcdir=None, topobjdir=None, dist_dir=None): + """Create a WebIDLCodegenManager for use by the build system.""" + if topsrcdir is None: + assert topobjdir is None and dist_dir is None + import buildconfig + + topsrcdir = buildconfig.topsrcdir + topobjdir = buildconfig.topobjdir + dist_dir = buildconfig.substs["DIST"] + + src_dir = os.path.join(topsrcdir, "dom", "bindings") + obj_dir = os.path.join(topobjdir, "dom", "bindings") + webidl_root = os.path.join(topsrcdir, "dom", "webidl") + + with io.open(os.path.join(obj_dir, "file-lists.json"), "r") as fh: + files = json.load(fh) + + inputs = ( + files["webidls"], + files["exported_stems"], + files["generated_events_stems"], + files["example_interfaces"], + ) + + cache_dir = os.path.join(obj_dir, "_cache") + try: + os.makedirs(cache_dir) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + return WebIDLCodegenManager( + os.path.join(src_dir, "Bindings.conf"), + webidl_root, + inputs, + os.path.join(dist_dir, "include", "mozilla", "dom"), + obj_dir, + os.path.join(obj_dir, "codegen.json"), + cache_dir=cache_dir, + # The make rules include a codegen.pp file containing dependencies. + make_deps_path=os.path.join(obj_dir, "codegen.pp"), + make_deps_target="webidl.stub", + ) diff --git a/dom/bindings/mozwebidlcodegen/test/Child.webidl b/dom/bindings/mozwebidlcodegen/test/Child.webidl new file mode 100644 index 0000000000..aa400a52a1 --- /dev/null +++ b/dom/bindings/mozwebidlcodegen/test/Child.webidl @@ -0,0 +1,3 @@ +interface Child : Parent { + undefined ChildBaz(); +}; diff --git a/dom/bindings/mozwebidlcodegen/test/ExampleBinding.webidl b/dom/bindings/mozwebidlcodegen/test/ExampleBinding.webidl new file mode 100644 index 0000000000..34794993fe --- /dev/null +++ b/dom/bindings/mozwebidlcodegen/test/ExampleBinding.webidl @@ -0,0 +1,3 @@ +/* These interfaces are hard-coded and need to be defined. */ +interface TestExampleInterface {}; +interface TestExampleProxyInterface {}; diff --git a/dom/bindings/mozwebidlcodegen/test/Parent.webidl b/dom/bindings/mozwebidlcodegen/test/Parent.webidl new file mode 100644 index 0000000000..9581a6b4e7 --- /dev/null +++ b/dom/bindings/mozwebidlcodegen/test/Parent.webidl @@ -0,0 +1,3 @@ +interface Parent { + undefined MethodFoo(); +}; diff --git a/dom/bindings/mozwebidlcodegen/test/TestEvent.webidl b/dom/bindings/mozwebidlcodegen/test/TestEvent.webidl new file mode 100644 index 0000000000..0b795a8024 --- /dev/null +++ b/dom/bindings/mozwebidlcodegen/test/TestEvent.webidl @@ -0,0 +1,13 @@ +interface EventTarget { + undefined addEventListener(); +}; + +interface Event {}; + +callback EventHandlerNonNull = any (Event event); +typedef EventHandlerNonNull? EventHandler; + +[LegacyNoInterfaceObject] +interface TestEvent : EventTarget { + attribute EventHandler onfoo; +}; diff --git a/dom/bindings/mozwebidlcodegen/test/python.ini b/dom/bindings/mozwebidlcodegen/test/python.ini new file mode 100644 index 0000000000..3b86feafb3 --- /dev/null +++ b/dom/bindings/mozwebidlcodegen/test/python.ini @@ -0,0 +1,4 @@ +[DEFAULT] +subsuite = mozbuild + +[test_mozwebidlcodegen.py] diff --git a/dom/bindings/mozwebidlcodegen/test/test_mozwebidlcodegen.py b/dom/bindings/mozwebidlcodegen/test/test_mozwebidlcodegen.py new file mode 100644 index 0000000000..10129022e6 --- /dev/null +++ b/dom/bindings/mozwebidlcodegen/test/test_mozwebidlcodegen.py @@ -0,0 +1,297 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import imp +import io +import json +import os +import shutil +import sys +import tempfile +import unittest + +import mozpack.path as mozpath +from mozfile import NamedTemporaryFile +from mozunit import MockedOpen, main +from mozwebidlcodegen import WebIDLCodegenManager, WebIDLCodegenManagerState + +OUR_DIR = mozpath.abspath(mozpath.dirname(__file__)) +TOPSRCDIR = mozpath.normpath(mozpath.join(OUR_DIR, "..", "..", "..", "..")) + + +class TestWebIDLCodegenManager(unittest.TestCase): + TEST_STEMS = { + "Child", + "Parent", + "ExampleBinding", + "TestEvent", + } + + @property + def _static_input_paths(self): + s = { + mozpath.join(OUR_DIR, p) + for p in os.listdir(OUR_DIR) + if p.endswith(".webidl") + } + + return s + + @property + def _config_path(self): + config = mozpath.join(TOPSRCDIR, "dom", "bindings", "Bindings.conf") + self.assertTrue(os.path.exists(config)) + + return config + + def _get_manager_args(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp) + + cache_dir = mozpath.join(tmp, "cache") + os.mkdir(cache_dir) + + ip = self._static_input_paths + + inputs = ( + ip, + {mozpath.splitext(mozpath.basename(p))[0] for p in ip}, + set(), + set(), + ) + + return dict( + config_path=self._config_path, + webidl_root=cache_dir, + inputs=inputs, + exported_header_dir=mozpath.join(tmp, "exports"), + codegen_dir=mozpath.join(tmp, "codegen"), + state_path=mozpath.join(tmp, "state.json"), + make_deps_path=mozpath.join(tmp, "codegen.pp"), + make_deps_target="codegen.pp", + cache_dir=cache_dir, + ) + + def _get_manager(self): + return WebIDLCodegenManager(**self._get_manager_args()) + + def test_unknown_state_version(self): + """Loading a state file with a too new version resets state.""" + args = self._get_manager_args() + + p = args["state_path"] + + with io.open(p, "w", newline="\n") as fh: + json.dump( + { + "version": WebIDLCodegenManagerState.VERSION + 1, + "foobar": "1", + }, + fh, + ) + + manager = WebIDLCodegenManager(**args) + + self.assertEqual(manager._state["version"], WebIDLCodegenManagerState.VERSION) + self.assertNotIn("foobar", manager._state) + + def test_generate_build_files(self): + """generate_build_files() does the right thing from empty.""" + manager = self._get_manager() + result = manager.generate_build_files() + self.assertEqual(len(result.inputs), 4) + + output = manager.expected_build_output_files() + self.assertEqual(result.created, output) + self.assertEqual(len(result.updated), 0) + self.assertEqual(len(result.unchanged), 0) + + for f in output: + self.assertTrue(os.path.isfile(f)) + + for f in manager.GLOBAL_DECLARE_FILES: + self.assertIn(mozpath.join(manager._exported_header_dir, f), output) + + for f in manager.GLOBAL_DEFINE_FILES: + self.assertIn(mozpath.join(manager._codegen_dir, f), output) + + for s in self.TEST_STEMS: + self.assertTrue( + os.path.isfile( + mozpath.join(manager._exported_header_dir, "%sBinding.h" % s) + ) + ) + self.assertTrue( + os.path.isfile(mozpath.join(manager._codegen_dir, "%sBinding.cpp" % s)) + ) + + self.assertTrue(os.path.isfile(manager._state_path)) + + with io.open(manager._state_path, "r") as fh: + state = json.load(fh) + self.assertEqual(state["version"], 3) + self.assertIn("webidls", state) + + child = state["webidls"]["Child.webidl"] + self.assertEqual(len(child["inputs"]), 2) + self.assertEqual(len(child["outputs"]), 2) + self.assertEqual(child["sha1"], "c34c40b0fa0ac57c2834ee282efe0681e4dacc35") + + def test_generate_build_files_load_state(self): + """State should be equivalent when instantiating a new instance.""" + args = self._get_manager_args() + m1 = WebIDLCodegenManager(**args) + self.assertEqual(len(m1._state["webidls"]), 0) + m1.generate_build_files() + + m2 = WebIDLCodegenManager(**args) + self.assertGreater(len(m2._state["webidls"]), 2) + self.assertEqual(m1._state, m2._state) + + def test_no_change_no_writes(self): + """If nothing changes, no files should be updated.""" + args = self._get_manager_args() + m1 = WebIDLCodegenManager(**args) + m1.generate_build_files() + + m2 = WebIDLCodegenManager(**args) + result = m2.generate_build_files() + + self.assertEqual(len(result.inputs), 0) + self.assertEqual(len(result.created), 0) + self.assertEqual(len(result.updated), 0) + + def test_output_file_regenerated(self): + """If an output file disappears, it is regenerated.""" + args = self._get_manager_args() + m1 = WebIDLCodegenManager(**args) + m1.generate_build_files() + + rm_count = 0 + for p in m1._state["webidls"]["Child.webidl"]["outputs"]: + rm_count += 1 + os.unlink(p) + + for p in m1.GLOBAL_DECLARE_FILES: + rm_count += 1 + os.unlink(mozpath.join(m1._exported_header_dir, p)) + + m2 = WebIDLCodegenManager(**args) + result = m2.generate_build_files() + self.assertEqual(len(result.created), rm_count) + + def test_only_rebuild_self(self): + """If an input file changes, only rebuild that one file.""" + args = self._get_manager_args() + m1 = WebIDLCodegenManager(**args) + m1.generate_build_files() + + child_path = None + for p in m1._input_paths: + if p.endswith("Child.webidl"): + child_path = p + break + + self.assertIsNotNone(child_path) + child_content = io.open(child_path, "r").read() + + with MockedOpen({child_path: child_content + "\n/* */"}): + m2 = WebIDLCodegenManager(**args) + result = m2.generate_build_files() + self.assertEqual(result.inputs, set([child_path])) + self.assertEqual(len(result.updated), 0) + self.assertEqual(len(result.created), 0) + + def test_rebuild_dependencies(self): + """Ensure an input file used by others results in others rebuilding.""" + args = self._get_manager_args() + m1 = WebIDLCodegenManager(**args) + m1.generate_build_files() + + parent_path = None + child_path = None + for p in m1._input_paths: + if p.endswith("Parent.webidl"): + parent_path = p + elif p.endswith("Child.webidl"): + child_path = p + + self.assertIsNotNone(parent_path) + parent_content = io.open(parent_path, "r").read() + + with MockedOpen({parent_path: parent_content + "\n/* */"}): + m2 = WebIDLCodegenManager(**args) + result = m2.generate_build_files() + self.assertEqual(result.inputs, {child_path, parent_path}) + self.assertEqual(len(result.updated), 0) + self.assertEqual(len(result.created), 0) + + def test_python_change_regenerate_everything(self): + """If a Python file changes, we should attempt to rebuild everything.""" + + # We don't want to mutate files in the source directory because we want + # to be able to build from a read-only filesystem. So, we install a + # dummy module and rewrite the metadata to say it comes from the source + # directory. + # + # Hacking imp to accept a MockedFile doesn't appear possible. So for + # the first iteration we read from a temp file. The second iteration + # doesn't need to import, so we are fine with a mocked file. + fake_path = mozpath.join(OUR_DIR, "fakemodule.py") + with NamedTemporaryFile("wt") as fh: + fh.write("# Original content") + fh.flush() + mod = imp.load_source("mozwebidlcodegen.fakemodule", fh.name) + mod.__file__ = fake_path + + args = self._get_manager_args() + m1 = WebIDLCodegenManager(**args) + with MockedOpen({fake_path: "# Original content"}): + try: + result = m1.generate_build_files() + l = len(result.inputs) + + with io.open(fake_path, "wt", newline="\n") as fh: + fh.write("# Modified content") + + m2 = WebIDLCodegenManager(**args) + result = m2.generate_build_files() + self.assertEqual(len(result.inputs), l) + + result = m2.generate_build_files() + self.assertEqual(len(result.inputs), 0) + finally: + del sys.modules["mozwebidlcodegen.fakemodule"] + + def test_copy_input(self): + """Ensure a copied .webidl file is handled properly.""" + + # This test simulates changing the type of a WebIDL from static to + # preprocessed. In that scenario, the original file still exists but + # it should no longer be consulted during codegen. + + args = self._get_manager_args() + m1 = WebIDLCodegenManager(**args) + m1.generate_build_files() + + old_path = None + for p in args["inputs"][0]: + if p.endswith("Parent.webidl"): + old_path = p + break + self.assertIsNotNone(old_path) + + new_path = mozpath.join(args["cache_dir"], "Parent.webidl") + shutil.copy2(old_path, new_path) + + args["inputs"][0].remove(old_path) + args["inputs"][0].add(new_path) + + m2 = WebIDLCodegenManager(**args) + result = m2.generate_build_files() + self.assertEqual(len(result.updated), 0) + + +if __name__ == "__main__": + main() |