Merging upstream version 126.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

2 files changed, 78 insertions, 10 deletions

diff --git a/dom/bindings/mozwebidlcodegen/__init__.py b/dom/bindings/mozwebidlcodegen/__init__.py
index 3e8c6aa420..bba95edb78 100644
--- a/dom/bindings/mozwebidlcodegen/__init__.py
+++ b/dom/bindings/mozwebidlcodegen/__init__.py
@@ -11,7 +11,8 @@ import io
 import json
 import logging
 import os
-from copy import deepcopy
+import sys
+from multiprocessing import Pool
 
 import mozpack.path as mozpath
 from mach.mixin.logging import LoggingMixin
@@ -22,6 +23,50 @@ from mozbuild.util import FileAvoidWrite
 # There are various imports in this file in functions to avoid adding
 # dependencies to config.status. See bug 949875.
 
+# Limit the count on Windows, because of bug 1889842 and also the
+# inefficiency of fork on Windows.
+DEFAULT_PROCESS_COUNT = 4 if sys.platform == "win32" else os.cpu_count()
+
+
+class WebIDLPool:
+    """
+    Distribute generation load across several processes, avoiding redundant state
+    copies.
+    """
+
+    GeneratorState = None
+
+    def __init__(self, GeneratorState, *, processes=None):
+        if processes is None:
+            processes = DEFAULT_PROCESS_COUNT
+
+        # As a special case, don't spawn an extra process if processes=1
+        if processes == 1:
+            WebIDLPool._init(GeneratorState)
+
+            class SeqPool:
+                def map(self, *args):
+                    return list(map(*args))
+
+            self.pool = SeqPool()
+        else:
+            self.pool = Pool(
+                initializer=WebIDLPool._init,
+                initargs=(GeneratorState,),
+                processes=processes,
+            )
+
+    def run(self, filenames):
+        return self.pool.map(WebIDLPool._run, filenames)
+
+    @staticmethod
+    def _init(GeneratorState):
+        WebIDLPool.GeneratorState = GeneratorState
+
+    @staticmethod
+    def _run(filename):
+        return WebIDLPool.GeneratorState._generate_build_files_for_webidl(filename)
+
 
 class BuildResult(object):
     """Represents the result of processing WebIDL files.
@@ -50,6 +95,9 @@ class WebIDLCodegenManagerState(dict):
     state should be considered a black box to everyone except
     WebIDLCodegenManager. But we'll still document it.
 
+    Any set stored in this dict should be copied and sorted in the `dump()`
+    method.
+
     Fields:
 
     version
@@ -117,12 +165,15 @@ class WebIDLCodegenManagerState(dict):
 
     def dump(self, fh):
         """Dump serialized state to a file handle."""
-        normalized = deepcopy(self)
+        normalized = self.copy()
 
+        webidls = normalized["webidls"] = self["webidls"].copy()
         for k, v in self["webidls"].items():
+            webidls_k = webidls[k] = v.copy()
+
             # Convert sets to lists because JSON doesn't support sets.
-            normalized["webidls"][k]["outputs"] = sorted(v["outputs"])
-            normalized["webidls"][k]["inputs"] = sorted(v["inputs"])
+            webidls_k["outputs"] = sorted(v["outputs"])
+            webidls_k["inputs"] = sorted(v["inputs"])
 
         normalized["dictionaries_convertible_to_js"] = sorted(
             self["dictionaries_convertible_to_js"]
@@ -251,7 +302,7 @@ class WebIDLCodegenManager(LoggingMixin):
 
         return self._config
 
-    def generate_build_files(self):
+    def generate_build_files(self, *, processes=None):
         """Generate files required for the build.
 
         This function is in charge of generating all the .h/.cpp files derived
@@ -279,6 +330,9 @@ class WebIDLCodegenManager(LoggingMixin):
            file.
         3. If an output file is missing, ensure it is present by performing
            necessary regeneration.
+
+        if `processes` is set to None, run in parallel using the
+        multiprocess.Pool default. If set to 1, don't use extra processes.
         """
         # Despite #1 above, we assume the build system is smart enough to not
         # invoke us if nothing has changed. Therefore, any invocation means
@@ -311,11 +365,20 @@ class WebIDLCodegenManager(LoggingMixin):
             d.identifier.name for d in self._config.getDictionariesConvertibleFromJS()
         )
 
+        # Distribute the generation load across several processes. This requires
+        # a) that `self' is serializable and b) that `self' is unchanged by
+        # _generate_build_files_for_webidl(...)
+        ordered_changed_inputs = sorted(changed_inputs)
+        pool = WebIDLPool(self, processes=processes)
+        generation_results = pool.run(ordered_changed_inputs)
+
         # Generate bindings from .webidl files.
-        for filename in sorted(changed_inputs):
+        for filename, generation_result in zip(
+            ordered_changed_inputs, generation_results
+        ):
             basename = mozpath.basename(filename)
             result.inputs.add(filename)
-            written, deps = self._generate_build_files_for_webidl(filename)
+            written, deps = generation_result
             result.created |= written[0]
             result.updated |= written[1]
             result.unchanged |= written[2]
@@ -560,6 +623,8 @@ class WebIDLCodegenManager(LoggingMixin):
 
         return paths
 
+    # Parallelization of the generation step relies on this method not changing
+    # the internal state of the object
     def _generate_build_files_for_webidl(self, filename):
         from Codegen import CGBindingRoot, CGEventRoot
 
diff --git a/dom/bindings/mozwebidlcodegen/test/test_mozwebidlcodegen.py b/dom/bindings/mozwebidlcodegen/test/test_mozwebidlcodegen.py
index cd822af538..46a3ab6239 100644
--- a/dom/bindings/mozwebidlcodegen/test/test_mozwebidlcodegen.py
+++ b/dom/bindings/mozwebidlcodegen/test/test_mozwebidlcodegen.py
@@ -247,18 +247,21 @@ class TestWebIDLCodegenManager(unittest.TestCase):
             args = self._get_manager_args()
             m1 = WebIDLCodegenManager(**args)
             with MockedOpen({fake_path: "# Original content"}):
+                # MockedOpen is not compatible with distributed filesystem
+                # access, so force the number of processes used to generate
+                # files to 1.
                 try:
-                    result = m1.generate_build_files()
+                    result = m1.generate_build_files(processes=1)
                     l = len(result.inputs)
 
                     with io.open(fake_path, "wt", newline="\n") as fh:
                         fh.write("# Modified content")
 
                     m2 = WebIDLCodegenManager(**args)
-                    result = m2.generate_build_files()
+                    result = m2.generate_build_files(processes=1)
                     self.assertEqual(len(result.inputs), l)
 
-                    result = m2.generate_build_files()
+                    result = m2.generate_build_files(processes=1)
                     self.assertEqual(len(result.inputs), 0)
                 finally:
                     del sys.modules["mozwebidlcodegen.fakemodule"]