Adding upstream version 127.0.upstream/127.0

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 333 insertions, 252 deletions

diff --git a/ipc/ipdl/ipdl.py b/ipc/ipdl/ipdl.py
index 8e5dd5db3d..5c4b7a8344 100644
--- a/ipc/ipdl/ipdl.py
+++ b/ipc/ipdl/ipdl.py
@@ -6,214 +6,291 @@ import os
 import sys
 from configparser import RawConfigParser
 from io import StringIO
+from multiprocessing import Manager
 
 import ipdl
 from ipdl.ast import SYNC
 
 
-def log(minv, fmt, *args):
-    if _verbosity >= minv:
-        print(fmt % args)
-
-
-# process command line
-
-
-op = optparse.OptionParser(usage="ipdl.py [options] IPDLfiles...")
-op.add_option(
-    "-I",
-    "--include",
-    dest="includedirs",
-    default=[],
-    action="append",
-    help="Additional directory to search for included protocol specifications",
-)
-op.add_option(
-    "-s",
-    "--sync-msg-list",
-    dest="syncMsgList",
-    default="sync-messages.ini",
-    help="Config file listing allowed sync messages",
-)
-op.add_option(
-    "-m",
-    "--msg-metadata",
-    dest="msgMetadata",
-    default="message-metadata.ini",
-    help="Predicted message sizes for reducing serialization malloc overhead.",
-)
-op.add_option(
-    "-v",
-    "--verbose",
-    dest="verbosity",
-    default=1,
-    action="count",
-    help="Verbose logging (specify -vv or -vvv for very verbose logging)",
-)
-op.add_option(
-    "-q",
-    "--quiet",
-    dest="verbosity",
-    action="store_const",
-    const=0,
-    help="Suppress logging output",
-)
-op.add_option(
-    "-d",
-    "--outheaders-dir",
-    dest="headersdir",
-    default=".",
-    help="""Directory into which C++ headers will be generated.
-A protocol Foo in the namespace bar will cause the headers
-  dir/bar/Foo.h, dir/bar/FooParent.h, and dir/bar/FooParent.h
-to be generated""",
-)
-op.add_option(
-    "-o",
-    "--outcpp-dir",
-    dest="cppdir",
-    default=".",
-    help="""Directory into which C++ sources will be generated
-A protocol Foo in the namespace bar will cause the sources
-  cppdir/FooParent.cpp, cppdir/FooChild.cpp
-to be generated""",
-)
-op.add_option(
-    "-F",
-    "--file-list",
-    dest="file_list_file",
-    default=None,
-    help="""A file containing IPDL files to parse. This will be
-merged with files provided on the commandline.""",
-)
-
-options, cmdline_files = op.parse_args()
-_verbosity = options.verbosity
-syncMsgList = options.syncMsgList
-msgMetadata = options.msgMetadata
-headersdir = options.headersdir
-cppdir = options.cppdir
-includedirs = [os.path.abspath(incdir) for incdir in options.includedirs]
-
-files = []
-
-if options.file_list_file is not None:
-    with open(options.file_list_file) as f:
-        files.extend(f.read().splitlines())
-
-files.extend(cmdline_files)
-
-if not len(files):
-    op.error("No IPDL files specified")
-
-ipcmessagestartpath = os.path.join(headersdir, "IPCMessageStart.h")
-ipc_msgtype_name_path = os.path.join(cppdir, "IPCMessageTypeName.cpp")
-
-log(2, 'Generated C++ headers will be generated relative to "%s"', headersdir)
-log(2, 'Generated C++ sources will be generated in "%s"', cppdir)
-
-allmessages = {}
-allsyncmessages = []
-allmessageprognames = []
-allprotocols = []
-
-
-def normalizedFilename(f):
-    if f == "-":
-        return "<stdin>"
-    return f
-
-
-log(2, "Reading sync message list")
-parser = RawConfigParser()
-parser.read_file(open(options.syncMsgList))
-syncMsgList = parser.sections()
-
-for section in syncMsgList:
-    if not parser.get(section, "description"):
-        print("Error: Sync message %s lacks a description" % section, file=sys.stderr)
-        sys.exit(1)
-
-# Read message metadata. Right now we only have 'segment_capacity'
-# for the standard segment size used for serialization.
-log(2, "Reading message metadata...")
-msgMetadataConfig = RawConfigParser()
-msgMetadataConfig.read_file(open(options.msgMetadata))
-
-segmentCapacityDict = {}
-for msgName in msgMetadataConfig.sections():
-    if msgMetadataConfig.has_option(msgName, "segment_capacity"):
-        capacity = msgMetadataConfig.get(msgName, "segment_capacity")
-        segmentCapacityDict[msgName] = capacity
-
-# First pass: parse and type-check all protocols
-for f in files:
-    log(2, os.path.basename(f))
-    filename = normalizedFilename(f)
-    if f == "-":
-        fd = sys.stdin
-    else:
-        fd = open(f)
-
-    specstring = fd.read()
-    fd.close()
-
-    ast = ipdl.parse(specstring, filename, includedirs=includedirs)
-    if ast is None:
-        print("Specification could not be parsed.", file=sys.stderr)
-        sys.exit(1)
+class WorkerPool:
+    per_process_context = None
+
+    def __init__(
+        self,
+        manager,
+        files,
+        asts,
+        headersdir,
+        cppdir,
+        segmentCapacityDict,
+        allmessages,
+        allprotocols,
+        allmessageprognames,
+        allsyncmessages,
+        *,
+        processes=None
+    ):
+        if processes is None:
+            processes = os.cpu_count() or 1
+        processes = min(processes, 8)
+        self.n = len(files)
+        self.asts = asts
+        self.pool = manager.Pool(
+            initializer=WorkerPool._init_worker,
+            initargs=(
+                files,
+                asts,
+                headersdir,
+                cppdir,
+                segmentCapacityDict,
+                allmessages,
+                allprotocols,
+                allmessageprognames,
+                allsyncmessages,
+            ),
+            processes=processes,
+        )
 
-    log(2, "checking types")
-    if not ipdl.typecheck(ast):
-        print("Specification is not well typed.", file=sys.stderr)
+    def run(self):
+        self.pool.map(WorkerPool._run_worker, range(self.n))
+
+    @staticmethod
+    def _init_worker(*state):
+        WorkerPool.per_process_context = state
+
+    @staticmethod
+    def _run_worker(index):
+        (
+            files,
+            asts,
+            headersdir,
+            cppdir,
+            segmentCapacityDict,
+            allmessages,
+            allprotocols,
+            allmessageprognames,
+            allsyncmessages,
+        ) = WorkerPool.per_process_context
+        ast = asts[index]
+        ipdl.gencxx(files[index], ast, headersdir, cppdir, segmentCapacityDict)
+
+        if ast.protocol:
+            allmessages[ast.protocol.name] = ipdl.genmsgenum(ast)
+            allprotocols.append(ast.protocol.name)
+
+            # e.g. PContent::RequestMemoryReport (not prefixed or suffixed.)
+            for md in ast.protocol.messageDecls:
+                allmessageprognames.append("%s::%s" % (md.namespace, md.decl.progname))
+
+                if md.sendSemantics is SYNC:
+                    allsyncmessages.append(
+                        "%s__%s" % (ast.protocol.name, md.prettyMsgName())
+                    )
+
+
+def main():
+    def log(minv, fmt, *args):
+        if _verbosity >= minv:
+            print(fmt % args)
+
+    # process command line
+
+    op = optparse.OptionParser(usage="ipdl.py [options] IPDLfiles...")
+    op.add_option(
+        "-I",
+        "--include",
+        dest="includedirs",
+        default=[],
+        action="append",
+        help="Additional directory to search for included protocol specifications",
+    )
+    op.add_option(
+        "-s",
+        "--sync-msg-list",
+        dest="syncMsgList",
+        default="sync-messages.ini",
+        help="Config file listing allowed sync messages",
+    )
+    op.add_option(
+        "-m",
+        "--msg-metadata",
+        dest="msgMetadata",
+        default="message-metadata.ini",
+        help="Predicted message sizes for reducing serialization malloc overhead.",
+    )
+    op.add_option(
+        "-v",
+        "--verbose",
+        dest="verbosity",
+        default=1,
+        action="count",
+        help="Verbose logging (specify -vv or -vvv for very verbose logging)",
+    )
+    op.add_option(
+        "-q",
+        "--quiet",
+        dest="verbosity",
+        action="store_const",
+        const=0,
+        help="Suppress logging output",
+    )
+    op.add_option(
+        "-d",
+        "--outheaders-dir",
+        dest="headersdir",
+        default=".",
+        help="""Directory into which C++ headers will be generated.
+    A protocol Foo in the namespace bar will cause the headers
+      dir/bar/Foo.h, dir/bar/FooParent.h, and dir/bar/FooParent.h
+    to be generated""",
+    )
+    op.add_option(
+        "-o",
+        "--outcpp-dir",
+        dest="cppdir",
+        default=".",
+        help="""Directory into which C++ sources will be generated
+    A protocol Foo in the namespace bar will cause the sources
+      cppdir/FooParent.cpp, cppdir/FooChild.cpp
+    to be generated""",
+    )
+    op.add_option(
+        "-F",
+        "--file-list",
+        dest="file_list_file",
+        default=None,
+        help="""A file containing IPDL files to parse. This will be
+    merged with files provided on the commandline.""",
+    )
+
+    options, cmdline_files = op.parse_args()
+    _verbosity = options.verbosity
+    syncMsgList = options.syncMsgList
+    headersdir = options.headersdir
+    cppdir = options.cppdir
+    includedirs = [os.path.abspath(incdir) for incdir in options.includedirs]
+
+    files = []
+
+    if options.file_list_file is not None:
+        with open(options.file_list_file) as f:
+            files.extend(f.read().splitlines())
+
+    files.extend(cmdline_files)
+
+    if not len(files):
+        op.error("No IPDL files specified")
+
+    ipcmessagestartpath = os.path.join(headersdir, "IPCMessageStart.h")
+    ipc_msgtype_name_path = os.path.join(cppdir, "IPCMessageTypeName.cpp")
+
+    log(2, 'Generated C++ headers will be generated relative to "%s"', headersdir)
+    log(2, 'Generated C++ sources will be generated in "%s"', cppdir)
+
+    def normalizedFilename(f):
+        if f == "-":
+            return "<stdin>"
+        return f
+
+    log(2, "Reading sync message list")
+    parser = RawConfigParser()
+    parser.read_file(open(options.syncMsgList))
+    syncMsgList = parser.sections()
+
+    for section in syncMsgList:
+        if not parser.get(section, "description"):
+            print(
+                "Error: Sync message %s lacks a description" % section, file=sys.stderr
+            )
+            sys.exit(1)
+
+    # Read message metadata. Right now we only have 'segment_capacity'
+    # for the standard segment size used for serialization.
+    log(2, "Reading message metadata...")
+    msgMetadataConfig = RawConfigParser()
+    msgMetadataConfig.read_file(open(options.msgMetadata))
+
+    manager = Manager()
+    segmentCapacityDict = manager.dict()
+    allmessages = manager.dict()
+    allsyncmessages = manager.list()
+    allmessageprognames = manager.list()
+    allprotocols = manager.list()
+
+    for msgName in msgMetadataConfig.sections():
+        if msgMetadataConfig.has_option(msgName, "segment_capacity"):
+            capacity = msgMetadataConfig.get(msgName, "segment_capacity")
+            segmentCapacityDict[msgName] = capacity
+
+    # First pass: parse and type-check all protocols
+    asts = []
+    for f in files:
+        log(2, os.path.basename(f))
+        filename = normalizedFilename(f)
+        if f == "-":
+            fd = sys.stdin
+        else:
+            fd = open(f)
+
+        specstring = fd.read()
+        fd.close()
+
+        ast = ipdl.parse(specstring, filename, includedirs=includedirs)
+        if ast is None:
+            print("Specification could not be parsed.", file=sys.stderr)
+            sys.exit(1)
+
+        log(2, "checking types")
+        if not ipdl.typecheck(ast):
+            print("Specification is not well typed.", file=sys.stderr)
+            sys.exit(1)
+
+        if not ipdl.checkSyncMessage(ast, syncMsgList):
+            print(
+                "Error: New sync IPC messages must be reviewed by an IPC peer and recorded in %s"
+                % options.syncMsgList,
+                file=sys.stderr,
+            )  # NOQA: E501
+            sys.exit(1)
+
+        asts.append(ast)
+
+    if not ipdl.checkFixedSyncMessages(parser):
+        # Errors have alraedy been printed to stderr, just exit
         sys.exit(1)
 
-    if not ipdl.checkSyncMessage(ast, syncMsgList):
+    # Second pass: generate code
+    pool = WorkerPool(
+        manager,
+        files,
+        asts,
+        headersdir,
+        cppdir,
+        segmentCapacityDict,
+        allmessages,
+        allprotocols,
+        allmessageprognames,
+        allsyncmessages,
+    )
+    pool.run()
+
+    allprotocols.sort()
+    allsyncmessages.sort()
+
+    # Check if we have undefined message names in segmentCapacityDict.
+    # This is a fool-proof of the 'message-metadata.ini' file.
+    undefinedMessages = set(segmentCapacityDict.keys()) - set(allmessageprognames)
+    if len(undefinedMessages) > 0:
         print(
-            "Error: New sync IPC messages must be reviewed by an IPC peer and recorded in %s"
-            % options.syncMsgList,
-            file=sys.stderr,
-        )  # NOQA: E501
+            "Error: Undefined message names in message-metadata.ini:", file=sys.stderr
+        )
+        print(undefinedMessages, file=sys.stderr)
         sys.exit(1)
 
-if not ipdl.checkFixedSyncMessages(parser):
-    # Errors have alraedy been printed to stderr, just exit
-    sys.exit(1)
-
-# Second pass: generate code
-for f in files:
-    # Read from parser cache
-    filename = normalizedFilename(f)
-    ast = ipdl.parse(None, filename, includedirs=includedirs)
-    ipdl.gencxx(filename, ast, headersdir, cppdir, segmentCapacityDict)
-
-    if ast.protocol:
-        allmessages[ast.protocol.name] = ipdl.genmsgenum(ast)
-        allprotocols.append(ast.protocol.name)
-
-        # e.g. PContent::RequestMemoryReport (not prefixed or suffixed.)
-        for md in ast.protocol.messageDecls:
-            allmessageprognames.append("%s::%s" % (md.namespace, md.decl.progname))
-
-            if md.sendSemantics is SYNC:
-                allsyncmessages.append(
-                    "%s__%s" % (ast.protocol.name, md.prettyMsgName())
-                )
-
-allprotocols.sort()
+    ipcmsgstart = StringIO()
 
-# Check if we have undefined message names in segmentCapacityDict.
-# This is a fool-proof of the 'message-metadata.ini' file.
-undefinedMessages = set(segmentCapacityDict.keys()) - set(allmessageprognames)
-if len(undefinedMessages) > 0:
-    print("Error: Undefined message names in message-metadata.ini:", file=sys.stderr)
-    print(undefinedMessages, file=sys.stderr)
-    sys.exit(1)
-
-ipcmsgstart = StringIO()
-
-print(
-    """
+    print(
+        """
 // CODE GENERATED by ipdl.py. Do not edit.
 
 #ifndef IPCMessageStart_h
@@ -221,14 +298,14 @@ print(
 
 enum IPCMessageStart {
 """,
-    file=ipcmsgstart,
-)
+        file=ipcmsgstart,
+    )
 
-for name in allprotocols:
-    print("  %sMsgStart," % name, file=ipcmsgstart)
+    for name in allprotocols:
+        print("  %sMsgStart," % name, file=ipcmsgstart)
 
-print(
-    """
+    print(
+        """
   LastMsgIndex
 };
 
@@ -236,12 +313,12 @@ static_assert(LastMsgIndex <= 65536, "need to update IPC_MESSAGE_MACRO");
 
 #endif // ifndef IPCMessageStart_h
 """,
-    file=ipcmsgstart,
-)
+        file=ipcmsgstart,
+    )
 
-ipc_msgtype_name = StringIO()
-print(
-    """
+    ipc_msgtype_name = StringIO()
+    print(
+        """
 // CODE GENERATED by ipdl.py. Do not edit.
 #include <cstdint>
 
@@ -253,19 +330,19 @@ using std::uint32_t;
 namespace {
 
 enum IPCMessages {
-""",
-    file=ipc_msgtype_name,
-)
-
-for protocol in sorted(allmessages.keys()):
-    for msg, num in allmessages[protocol].idnums:
-        if num:
-            print("  %s = %s," % (msg, num), file=ipc_msgtype_name)
-        elif not msg.endswith("End"):
-            print("  %s__%s," % (protocol, msg), file=ipc_msgtype_name)
-
-print(
-    """
+    """,
+        file=ipc_msgtype_name,
+    )
+
+    for protocol in sorted(allmessages.keys()):
+        for msg, num in allmessages[protocol].idnums:
+            if num:
+                print("  %s = %s," % (msg, num), file=ipc_msgtype_name)
+            elif not msg.endswith("End"):
+                print("  %s__%s," % (protocol, msg), file=ipc_msgtype_name)
+
+    print(
+        """
 };
 
 } // anonymous namespace
@@ -276,14 +353,14 @@ bool IPCMessageTypeIsSync(uint32_t aMessageType)
 {
   switch (aMessageType) {
 """,
-    file=ipc_msgtype_name,
-)
+        file=ipc_msgtype_name,
+    )
 
-for msg in allsyncmessages:
-    print("  case %s:" % msg, file=ipc_msgtype_name)
+    for msg in allsyncmessages:
+        print("  case %s:" % msg, file=ipc_msgtype_name)
 
-print(
-    """    return true;
+    print(
+        """    return true;
   default:
     return false;
   }
@@ -292,24 +369,24 @@ print(
 const char* StringFromIPCMessageType(uint32_t aMessageType)
 {
   switch (aMessageType) {
-""",
-    file=ipc_msgtype_name,
-)
-
-for protocol in sorted(allmessages.keys()):
-    for msg, num in allmessages[protocol].idnums:
-        if num or msg.endswith("End"):
-            continue
-        print(
-            """
+    """,
+        file=ipc_msgtype_name,
+    )
+
+    for protocol in sorted(allmessages.keys()):
+        for msg, num in allmessages[protocol].idnums:
+            if num or msg.endswith("End"):
+                continue
+            print(
+                """
   case %s__%s:
     return "%s::%s";"""
-            % (protocol, msg, protocol, msg),
-            file=ipc_msgtype_name,
-        )
+                % (protocol, msg, protocol, msg),
+                file=ipc_msgtype_name,
+            )
 
-print(
-    """
+    print(
+        """
   case DATA_PIPE_CLOSED_MESSAGE_TYPE:
     return "DATA_PIPE_CLOSED_MESSAGE";
   case DATA_PIPE_BYTES_CONSUMED_MESSAGE_TYPE:
@@ -353,15 +430,15 @@ namespace ipc {
 const char* ProtocolIdToName(IPCMessageStart aId) {
   switch (aId) {
 """,
-    file=ipc_msgtype_name,
-)
+        file=ipc_msgtype_name,
+    )
 
-for name in allprotocols:
-    print("    case %sMsgStart:" % name, file=ipc_msgtype_name)
-    print('      return "%s";' % name, file=ipc_msgtype_name)
+    for name in allprotocols:
+        print("    case %sMsgStart:" % name, file=ipc_msgtype_name)
+        print('      return "%s";' % name, file=ipc_msgtype_name)
 
-print(
-    """
+    print(
+        """
   default:
     return "<unknown protocol id>";
   }
@@ -370,8 +447,12 @@ print(
 } // namespace ipc
 } // namespace mozilla
 """,
-    file=ipc_msgtype_name,
-)
+        file=ipc_msgtype_name,
+    )
+
+    ipdl.writeifmodified(ipcmsgstart.getvalue(), ipcmessagestartpath)
+    ipdl.writeifmodified(ipc_msgtype_name.getvalue(), ipc_msgtype_name_path)
+
 
-ipdl.writeifmodified(ipcmsgstart.getvalue(), ipcmessagestartpath)
-ipdl.writeifmodified(ipc_msgtype_name.getvalue(), ipc_msgtype_name_path)
+if __name__ == "__main__":
+    main()