diff options
Diffstat (limited to '')
| -rwxr-xr-x | src/VBox/Devices/EFI/Firmware/BaseTools/Source/Python/AutoGen/AutoGenWorker.py | 329 |
1 files changed, 329 insertions, 0 deletions
diff --git a/src/VBox/Devices/EFI/Firmware/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/src/VBox/Devices/EFI/Firmware/BaseTools/Source/Python/AutoGen/AutoGenWorker.py new file mode 100755 index 00000000..d392ffb2 --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/BaseTools/Source/Python/AutoGen/AutoGenWorker.py @@ -0,0 +1,329 @@ +## @file +# Create makefile for MS nmake and GNU make +# +# Copyright (c) 2019, Intel Corporation. All rights reserved.<BR> +# SPDX-License-Identifier: BSD-2-Clause-Patent +# +from __future__ import absolute_import +import multiprocessing as mp +import threading +from Common.Misc import PathClass +from AutoGen.ModuleAutoGen import ModuleAutoGen +from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo,AutoGenInfo +import Common.GlobalData as GlobalData +import Common.EdkLogger as EdkLogger +import os +from Common.MultipleWorkspace import MultipleWorkspace as mws +from AutoGen.AutoGen import AutoGen +from Workspace.WorkspaceDatabase import BuildDB +try: + from queue import Empty +except: + from Queue import Empty +import traceback +import sys +from AutoGen.DataPipe import MemoryDataPipe +import logging +import time + +def clearQ(q): + try: + while True: + q.get_nowait() + except Empty: + pass + +class LogAgent(threading.Thread): + def __init__(self,log_q,log_level,log_file=None): + super(LogAgent,self).__init__() + self.log_q = log_q + self.log_level = log_level + self.log_file = log_file + def InitLogger(self): + # For DEBUG level (All DEBUG_0~9 are applicable) + self._DebugLogger_agent = logging.getLogger("tool_debug_agent") + _DebugFormatter = logging.Formatter("[%(asctime)s.%(msecs)d]: %(message)s", datefmt="%H:%M:%S") + self._DebugLogger_agent.setLevel(self.log_level) + _DebugChannel = logging.StreamHandler(sys.stdout) + _DebugChannel.setFormatter(_DebugFormatter) + self._DebugLogger_agent.addHandler(_DebugChannel) + + # For VERBOSE, INFO, WARN level + self._InfoLogger_agent = logging.getLogger("tool_info_agent") + _InfoFormatter = logging.Formatter("%(message)s") + self._InfoLogger_agent.setLevel(self.log_level) + _InfoChannel = logging.StreamHandler(sys.stdout) + _InfoChannel.setFormatter(_InfoFormatter) + self._InfoLogger_agent.addHandler(_InfoChannel) + + # For ERROR level + self._ErrorLogger_agent = logging.getLogger("tool_error_agent") + _ErrorFormatter = logging.Formatter("%(message)s") + self._ErrorLogger_agent.setLevel(self.log_level) + _ErrorCh = logging.StreamHandler(sys.stderr) + _ErrorCh.setFormatter(_ErrorFormatter) + self._ErrorLogger_agent.addHandler(_ErrorCh) + + if self.log_file: + if os.path.exists(self.log_file): + os.remove(self.log_file) + _Ch = logging.FileHandler(self.log_file) + _Ch.setFormatter(_DebugFormatter) + self._DebugLogger_agent.addHandler(_Ch) + + _Ch= logging.FileHandler(self.log_file) + _Ch.setFormatter(_InfoFormatter) + self._InfoLogger_agent.addHandler(_Ch) + + _Ch = logging.FileHandler(self.log_file) + _Ch.setFormatter(_ErrorFormatter) + self._ErrorLogger_agent.addHandler(_Ch) + + def run(self): + self.InitLogger() + while True: + log_message = self.log_q.get() + if log_message is None: + break + if log_message.name == "tool_error": + self._ErrorLogger_agent.log(log_message.levelno,log_message.getMessage()) + elif log_message.name == "tool_info": + self._InfoLogger_agent.log(log_message.levelno,log_message.getMessage()) + elif log_message.name == "tool_debug": + self._DebugLogger_agent.log(log_message.levelno,log_message.getMessage()) + else: + self._InfoLogger_agent.log(log_message.levelno,log_message.getMessage()) + + def kill(self): + self.log_q.put(None) +class AutoGenManager(threading.Thread): + def __init__(self,autogen_workers, feedback_q,error_event): + super(AutoGenManager,self).__init__() + self.autogen_workers = autogen_workers + self.feedback_q = feedback_q + self.Status = True + self.error_event = error_event + def run(self): + try: + fin_num = 0 + while True: + badnews = self.feedback_q.get() + if badnews is None: + break + if badnews == "Done": + fin_num += 1 + elif badnews == "QueueEmpty": + EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), badnews)) + self.TerminateWorkers() + else: + EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), badnews)) + self.Status = False + self.TerminateWorkers() + if fin_num == len(self.autogen_workers): + self.clearQueue() + for w in self.autogen_workers: + w.join() + break + except Exception: + return + + def clearQueue(self): + taskq = self.autogen_workers[0].module_queue + logq = self.autogen_workers[0].log_q + clearQ(taskq) + clearQ(self.feedback_q) + clearQ(logq) + # Copy the cache queue itmes to parent thread before clear + cacheq = self.autogen_workers[0].cache_q + try: + cache_num = 0 + while True: + item = cacheq.get() + if item == "CacheDone": + cache_num += 1 + else: + GlobalData.gModuleAllCacheStatus.add(item) + if cache_num == len(self.autogen_workers): + break + except: + print ("cache_q error") + + def TerminateWorkers(self): + self.error_event.set() + def kill(self): + self.feedback_q.put(None) +class AutoGenWorkerInProcess(mp.Process): + def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,cache_q,log_q,error_event): + mp.Process.__init__(self) + self.module_queue = module_queue + self.data_pipe_file_path =data_pipe_file_path + self.data_pipe = None + self.feedback_q = feedback_q + self.PlatformMetaFileSet = {} + self.file_lock = file_lock + self.cache_q = cache_q + self.log_q = log_q + self.error_event = error_event + def GetPlatformMetaFile(self,filepath,root): + try: + return self.PlatformMetaFileSet[(filepath,root)] + except: + self.PlatformMetaFileSet[(filepath,root)] = filepath + return self.PlatformMetaFileSet[(filepath,root)] + def run(self): + try: + taskname = "Init" + with self.file_lock: + try: + self.data_pipe = MemoryDataPipe() + self.data_pipe.load(self.data_pipe_file_path) + except: + self.feedback_q.put(taskname + ":" + "load data pipe %s failed." % self.data_pipe_file_path) + EdkLogger.LogClientInitialize(self.log_q) + loglevel = self.data_pipe.Get("LogLevel") + if not loglevel: + loglevel = EdkLogger.INFO + EdkLogger.SetLevel(loglevel) + target = self.data_pipe.Get("P_Info").get("Target") + toolchain = self.data_pipe.Get("P_Info").get("ToolChain") + archlist = self.data_pipe.Get("P_Info").get("ArchList") + + active_p = self.data_pipe.Get("P_Info").get("ActivePlatform") + workspacedir = self.data_pipe.Get("P_Info").get("WorkspaceDir") + PackagesPath = os.getenv("PACKAGES_PATH") + mws.setWs(workspacedir, PackagesPath) + self.Wa = WorkSpaceInfo( + workspacedir,active_p,target,toolchain,archlist + ) + self.Wa._SrcTimeStamp = self.data_pipe.Get("Workspace_timestamp") + GlobalData.gGlobalDefines = self.data_pipe.Get("G_defines") + GlobalData.gCommandLineDefines = self.data_pipe.Get("CL_defines") + os.environ._data = self.data_pipe.Get("Env_Var") + GlobalData.gWorkspace = workspacedir + GlobalData.gDisableIncludePathCheck = False + GlobalData.gFdfParser = self.data_pipe.Get("FdfParser") + GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath") + + GlobalData.gUseHashCache = self.data_pipe.Get("UseHashCache") + GlobalData.gBinCacheSource = self.data_pipe.Get("BinCacheSource") + GlobalData.gBinCacheDest = self.data_pipe.Get("BinCacheDest") + GlobalData.gPlatformHashFile = self.data_pipe.Get("PlatformHashFile") + GlobalData.gModulePreMakeCacheStatus = dict() + GlobalData.gModuleMakeCacheStatus = dict() + GlobalData.gHashChainStatus = dict() + GlobalData.gCMakeHashFile = dict() + GlobalData.gModuleHashFile = dict() + GlobalData.gFileHashDict = dict() + GlobalData.gEnableGenfdsMultiThread = self.data_pipe.Get("EnableGenfdsMultiThread") + GlobalData.file_lock = self.file_lock + CommandTarget = self.data_pipe.Get("CommandTarget") + pcd_from_build_option = [] + for pcd_tuple in self.data_pipe.Get("BuildOptPcd"): + pcd_id = ".".join((pcd_tuple[0],pcd_tuple[1])) + if pcd_tuple[2].strip(): + pcd_id = ".".join((pcd_id,pcd_tuple[2])) + pcd_from_build_option.append("=".join((pcd_id,pcd_tuple[3]))) + GlobalData.BuildOptionPcd = pcd_from_build_option + module_count = 0 + FfsCmd = self.data_pipe.Get("FfsCommand") + if FfsCmd is None: + FfsCmd = {} + GlobalData.FfsCmd = FfsCmd + PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"), + self.data_pipe.Get("P_Info").get("WorkspaceDir")) + while True: + if self.error_event.is_set(): + break + module_count += 1 + try: + module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait() + except Empty: + EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Fake Empty.")) + time.sleep(0.01) + continue + if module_file is None: + EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Worker get the last item in the queue.")) + self.feedback_q.put("QueueEmpty") + time.sleep(0.01) + continue + + modulefullpath = os.path.join(module_root,module_file) + taskname = " : ".join((modulefullpath,module_arch)) + module_metafile = PathClass(module_file,module_root) + if module_path: + module_metafile.Path = module_path + if module_basename: + module_metafile.BaseName = module_basename + if module_originalpath: + module_metafile.OriginalPath = PathClass(module_originalpath,module_root) + arch = module_arch + target = self.data_pipe.Get("P_Info").get("Target") + toolchain = self.data_pipe.Get("P_Info").get("ToolChain") + Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe) + Ma.IsLibrary = IsLib + # SourceFileList calling sequence impact the makefile string sequence. + # Create cached SourceFileList here to unify its calling sequence for both + # CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile. + RetVal = Ma.SourceFileList + if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and CommandTarget in [None, "", "all"]: + try: + CacheResult = Ma.CanSkipbyPreMakeCache() + except: + CacheResult = False + self.feedback_q.put(taskname) + + if CacheResult: + self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", True)) + continue + else: + self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", False)) + + Ma.CreateCodeFile(False) + Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.Path, Ma.Arch),[])) + Ma.CreateAsBuiltInf() + if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: + try: + CacheResult = Ma.CanSkipbyMakeCache() + except: + CacheResult = False + self.feedback_q.put(taskname) + + if CacheResult: + self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", True)) + continue + else: + self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", False)) + + except Exception as e: + EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), str(e))) + self.feedback_q.put(taskname) + finally: + EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Done")) + self.feedback_q.put("Done") + self.cache_q.put("CacheDone") + + def printStatus(self): + print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache()))) + print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache()))) + groupobj = {} + for buildobj in BuildDB.BuildObject.GetCache().values(): + if str(buildobj).lower().endswith("dec"): + try: + groupobj['dec'].append(str(buildobj)) + except: + groupobj['dec'] = [str(buildobj)] + if str(buildobj).lower().endswith("dsc"): + try: + groupobj['dsc'].append(str(buildobj)) + except: + groupobj['dsc'] = [str(buildobj)] + + if str(buildobj).lower().endswith("inf"): + try: + groupobj['inf'].append(str(buildobj)) + except: + groupobj['inf'] = [str(buildobj)] + + print("Processs ID: %d Run %d pkg in WDB " % (os.getpid(),len(groupobj.get("dec",[])))) + print("Processs ID: %d Run %d pla in WDB " % (os.getpid(),len(groupobj.get("dsc",[])))) + print("Processs ID: %d Run %d inf in WDB " % (os.getpid(),len(groupobj.get("inf",[])))) |