diff options
Diffstat (limited to 'dom/bindings/mozwebidlcodegen/__init__.py')
| -rw-r--r-- | dom/bindings/mozwebidlcodegen/__init__.py | 79 |
1 files changed, 72 insertions, 7 deletions
diff --git a/dom/bindings/mozwebidlcodegen/__init__.py b/dom/bindings/mozwebidlcodegen/__init__.py index 3e8c6aa420..bba95edb78 100644 --- a/dom/bindings/mozwebidlcodegen/__init__.py +++ b/dom/bindings/mozwebidlcodegen/__init__.py @@ -11,7 +11,8 @@ import io import json import logging import os -from copy import deepcopy +import sys +from multiprocessing import Pool import mozpack.path as mozpath from mach.mixin.logging import LoggingMixin @@ -22,6 +23,50 @@ from mozbuild.util import FileAvoidWrite # There are various imports in this file in functions to avoid adding # dependencies to config.status. See bug 949875. +# Limit the count on Windows, because of bug 1889842 and also the +# inefficiency of fork on Windows. +DEFAULT_PROCESS_COUNT = 4 if sys.platform == "win32" else os.cpu_count() + + +class WebIDLPool: + """ + Distribute generation load across several processes, avoiding redundant state + copies. + """ + + GeneratorState = None + + def __init__(self, GeneratorState, *, processes=None): + if processes is None: + processes = DEFAULT_PROCESS_COUNT + + # As a special case, don't spawn an extra process if processes=1 + if processes == 1: + WebIDLPool._init(GeneratorState) + + class SeqPool: + def map(self, *args): + return list(map(*args)) + + self.pool = SeqPool() + else: + self.pool = Pool( + initializer=WebIDLPool._init, + initargs=(GeneratorState,), + processes=processes, + ) + + def run(self, filenames): + return self.pool.map(WebIDLPool._run, filenames) + + @staticmethod + def _init(GeneratorState): + WebIDLPool.GeneratorState = GeneratorState + + @staticmethod + def _run(filename): + return WebIDLPool.GeneratorState._generate_build_files_for_webidl(filename) + class BuildResult(object): """Represents the result of processing WebIDL files. @@ -50,6 +95,9 @@ class WebIDLCodegenManagerState(dict): state should be considered a black box to everyone except WebIDLCodegenManager. But we'll still document it. + Any set stored in this dict should be copied and sorted in the `dump()` + method. + Fields: version @@ -117,12 +165,15 @@ class WebIDLCodegenManagerState(dict): def dump(self, fh): """Dump serialized state to a file handle.""" - normalized = deepcopy(self) + normalized = self.copy() + webidls = normalized["webidls"] = self["webidls"].copy() for k, v in self["webidls"].items(): + webidls_k = webidls[k] = v.copy() + # Convert sets to lists because JSON doesn't support sets. - normalized["webidls"][k]["outputs"] = sorted(v["outputs"]) - normalized["webidls"][k]["inputs"] = sorted(v["inputs"]) + webidls_k["outputs"] = sorted(v["outputs"]) + webidls_k["inputs"] = sorted(v["inputs"]) normalized["dictionaries_convertible_to_js"] = sorted( self["dictionaries_convertible_to_js"] @@ -251,7 +302,7 @@ class WebIDLCodegenManager(LoggingMixin): return self._config - def generate_build_files(self): + def generate_build_files(self, *, processes=None): """Generate files required for the build. This function is in charge of generating all the .h/.cpp files derived @@ -279,6 +330,9 @@ class WebIDLCodegenManager(LoggingMixin): file. 3. If an output file is missing, ensure it is present by performing necessary regeneration. + + if `processes` is set to None, run in parallel using the + multiprocess.Pool default. If set to 1, don't use extra processes. """ # Despite #1 above, we assume the build system is smart enough to not # invoke us if nothing has changed. Therefore, any invocation means @@ -311,11 +365,20 @@ class WebIDLCodegenManager(LoggingMixin): d.identifier.name for d in self._config.getDictionariesConvertibleFromJS() ) + # Distribute the generation load across several processes. This requires + # a) that `self' is serializable and b) that `self' is unchanged by + # _generate_build_files_for_webidl(...) + ordered_changed_inputs = sorted(changed_inputs) + pool = WebIDLPool(self, processes=processes) + generation_results = pool.run(ordered_changed_inputs) + # Generate bindings from .webidl files. - for filename in sorted(changed_inputs): + for filename, generation_result in zip( + ordered_changed_inputs, generation_results + ): basename = mozpath.basename(filename) result.inputs.add(filename) - written, deps = self._generate_build_files_for_webidl(filename) + written, deps = generation_result result.created |= written[0] result.updated |= written[1] result.unchanged |= written[2] @@ -560,6 +623,8 @@ class WebIDLCodegenManager(LoggingMixin): return paths + # Parallelization of the generation step relies on this method not changing + # the internal state of the object def _generate_build_files_for_webidl(self, filename): from Codegen import CGBindingRoot, CGEventRoot |