diff options
Diffstat (limited to 'src/VBox/Devices/EFI/Firmware/BaseTools/Source/Python/build/build.py')
| -rwxr-xr-x | src/VBox/Devices/EFI/Firmware/BaseTools/Source/Python/build/build.py | 2796 |
1 files changed, 2796 insertions, 0 deletions
diff --git a/src/VBox/Devices/EFI/Firmware/BaseTools/Source/Python/build/build.py b/src/VBox/Devices/EFI/Firmware/BaseTools/Source/Python/build/build.py new file mode 100755 index 00000000..3790fbef --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/BaseTools/Source/Python/build/build.py @@ -0,0 +1,2796 @@ +## @file +# build a platform or a module +# +# Copyright (c) 2014, Hewlett-Packard Development Company, L.P.<BR> +# Copyright (c) 2007 - 2021, Intel Corporation. All rights reserved.<BR> +# Copyright (c) 2018, Hewlett Packard Enterprise Development, L.P.<BR> +# Copyright (c) 2020, ARM Limited. All rights reserved.<BR> +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# + +## +# Import Modules +# +from __future__ import print_function +from __future__ import absolute_import +import os.path as path +import sys +import os +import re +import glob +import time +import platform +import traceback +import multiprocessing +from threading import Thread,Event,BoundedSemaphore +import threading +from linecache import getlines +from subprocess import Popen,PIPE, STDOUT +from collections import OrderedDict, defaultdict + +from AutoGen.PlatformAutoGen import PlatformAutoGen +from AutoGen.ModuleAutoGen import ModuleAutoGen +from AutoGen.WorkspaceAutoGen import WorkspaceAutoGen +from AutoGen.AutoGenWorker import AutoGenWorkerInProcess,AutoGenManager,\ + LogAgent +from AutoGen import GenMake +from Common import Misc as Utils + +from Common.TargetTxtClassObject import TargetTxtDict +from Common.ToolDefClassObject import ToolDefDict +from buildoptions import MyOptionParser +from Common.Misc import PathClass,SaveFileOnChange,RemoveDirectory +from Common.StringUtils import NormPath +from Common.MultipleWorkspace import MultipleWorkspace as mws +from Common.BuildToolError import * +from Common.DataType import * +import Common.EdkLogger as EdkLogger + +from Workspace.WorkspaceDatabase import BuildDB + +from BuildReport import BuildReport +from GenPatchPcdTable.GenPatchPcdTable import PeImageClass,parsePcdInfoFromMapFile +from PatchPcdValue.PatchPcdValue import PatchBinaryFile + +import Common.GlobalData as GlobalData +from GenFds.GenFds import GenFds, GenFdsApi +import multiprocessing as mp +from multiprocessing import Manager +from AutoGen.DataPipe import MemoryDataPipe +from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo +from GenFds.FdfParser import FdfParser +from AutoGen.IncludesAutoGen import IncludesAutoGen +from GenFds.GenFds import resetFdsGlobalVariable +from AutoGen.AutoGen import CalculatePriorityValue + +## standard targets of build command +gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', 'fds', 'clean', 'cleanall', 'cleanlib', 'run'] + +## build configuration file +gBuildConfiguration = "target.txt" +gToolsDefinition = "tools_def.txt" + +TemporaryTablePattern = re.compile(r'^_\d+_\d+_[a-fA-F0-9]+$') +TmpTableDict = {} + +## Check environment PATH variable to make sure the specified tool is found +# +# If the tool is found in the PATH, then True is returned +# Otherwise, False is returned +# +def IsToolInPath(tool): + if 'PATHEXT' in os.environ: + extns = os.environ['PATHEXT'].split(os.path.pathsep) + else: + extns = ('',) + for pathDir in os.environ['PATH'].split(os.path.pathsep): + for ext in extns: + if os.path.exists(os.path.join(pathDir, tool + ext)): + return True + return False + +## Check environment variables +# +# Check environment variables that must be set for build. Currently they are +# +# WORKSPACE The directory all packages/platforms start from +# EDK_TOOLS_PATH The directory contains all tools needed by the build +# PATH $(EDK_TOOLS_PATH)/Bin/<sys> must be set in PATH +# +# If any of above environment variable is not set or has error, the build +# will be broken. +# +def CheckEnvVariable(): + # check WORKSPACE + if "WORKSPACE" not in os.environ: + EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found", + ExtraData="WORKSPACE") + + WorkspaceDir = os.path.normcase(os.path.normpath(os.environ["WORKSPACE"])) + if not os.path.exists(WorkspaceDir): + EdkLogger.error("build", FILE_NOT_FOUND, "WORKSPACE doesn't exist", ExtraData=WorkspaceDir) + elif ' ' in WorkspaceDir: + EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in WORKSPACE path", + ExtraData=WorkspaceDir) + os.environ["WORKSPACE"] = WorkspaceDir + + # set multiple workspace + PackagesPath = os.getenv("PACKAGES_PATH") + mws.setWs(WorkspaceDir, PackagesPath) + if mws.PACKAGES_PATH: + for Path in mws.PACKAGES_PATH: + if not os.path.exists(Path): + EdkLogger.error("build", FILE_NOT_FOUND, "One Path in PACKAGES_PATH doesn't exist", ExtraData=Path) + elif ' ' in Path: + EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in PACKAGES_PATH", ExtraData=Path) + + + os.environ["EDK_TOOLS_PATH"] = os.path.normcase(os.environ["EDK_TOOLS_PATH"]) + + # check EDK_TOOLS_PATH + if "EDK_TOOLS_PATH" not in os.environ: + EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found", + ExtraData="EDK_TOOLS_PATH") + + # check PATH + if "PATH" not in os.environ: + EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found", + ExtraData="PATH") + + GlobalData.gWorkspace = WorkspaceDir + + GlobalData.gGlobalDefines["WORKSPACE"] = WorkspaceDir + GlobalData.gGlobalDefines["EDK_TOOLS_PATH"] = os.environ["EDK_TOOLS_PATH"] + +## Get normalized file path +# +# Convert the path to be local format, and remove the WORKSPACE path at the +# beginning if the file path is given in full path. +# +# @param FilePath File path to be normalized +# @param Workspace Workspace path which the FilePath will be checked against +# +# @retval string The normalized file path +# +def NormFile(FilePath, Workspace): + # check if the path is absolute or relative + if os.path.isabs(FilePath): + FileFullPath = os.path.normpath(FilePath) + else: + FileFullPath = os.path.normpath(mws.join(Workspace, FilePath)) + Workspace = mws.getWs(Workspace, FilePath) + + # check if the file path exists or not + if not os.path.isfile(FileFullPath): + EdkLogger.error("build", FILE_NOT_FOUND, ExtraData="\t%s (Please give file in absolute path or relative to WORKSPACE)" % FileFullPath) + + # remove workspace directory from the beginning part of the file path + if Workspace[-1] in ["\\", "/"]: + return FileFullPath[len(Workspace):] + else: + return FileFullPath[(len(Workspace) + 1):] + +## Get the output of an external program +# +# This is the entrance method of thread reading output of an external program and +# putting them in STDOUT/STDERR of current program. +# +# @param From The stream message read from +# @param To The stream message put on +# @param ExitFlag The flag used to indicate stopping reading +# +def ReadMessage(From, To, ExitFlag,MemTo=None): + while True: + # read one line a time + Line = From.readline() + # empty string means "end" + if Line is not None and Line != b"": + LineStr = Line.rstrip().decode(encoding='utf-8', errors='ignore') + if MemTo is not None: + if "Note: including file:" == LineStr.lstrip()[:21]: + MemTo.append(LineStr) + else: + To(LineStr) + MemTo.append(LineStr) + else: + To(LineStr) + else: + break + if ExitFlag.isSet(): + break + +class MakeSubProc(Popen): + def __init__(self,*args, **argv): + super(MakeSubProc,self).__init__(*args, **argv) + self.ProcOut = [] + +## Launch an external program +# +# This method will call subprocess.Popen to execute an external program with +# given options in specified directory. Because of the dead-lock issue during +# redirecting output of the external program, threads are used to to do the +# redirection work. +# +# @param Command A list or string containing the call of the program +# @param WorkingDir The directory in which the program will be running +# +def LaunchCommand(Command, WorkingDir,ModuleAuto = None): + BeginTime = time.time() + # if working directory doesn't exist, Popen() will raise an exception + if not os.path.isdir(WorkingDir): + EdkLogger.error("build", FILE_NOT_FOUND, ExtraData=WorkingDir) + + # Command is used as the first Argument in following Popen(). + # It could be a string or sequence. We find that if command is a string in following Popen(), + # ubuntu may fail with an error message that the command is not found. + # So here we may need convert command from string to list instance. + if platform.system() != 'Windows': + if not isinstance(Command, list): + Command = Command.split() + Command = ' '.join(Command) + + if platform.system() != 'Windows': + EdkLogger.info("Launching: '%s'; CWD=%s" % (Command, WorkingDir)); + else: + EdkLogger.info("Launching: '%s'; CWD=%s" % ("' '".join(Command), WorkingDir)); + Proc = None + EndOfProcedure = None + try: + # launch the command + Proc = MakeSubProc(Command, stdout=PIPE, stderr=STDOUT, env=os.environ, cwd=WorkingDir, bufsize=-1, shell=True) + + # launch two threads to read the STDOUT and STDERR + EndOfProcedure = Event() + EndOfProcedure.clear() + if Proc.stdout: + StdOutThread = Thread(target=ReadMessage, args=(Proc.stdout, EdkLogger.info, EndOfProcedure,Proc.ProcOut)) + StdOutThread.setName("STDOUT-Redirector") + StdOutThread.setDaemon(False) + StdOutThread.start() + + + # waiting for program exit + Proc.wait() + except: # in case of aborting + # terminate the threads redirecting the program output + EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc()) + if EndOfProcedure is not None: + EndOfProcedure.set() + if Proc is None: + if not isinstance(Command, type("")): + Command = " ".join(Command) + EdkLogger.error("build", COMMAND_FAILURE, "Failed to start command", ExtraData="%s [%s]" % (Command, WorkingDir)) + + if Proc.stdout: + StdOutThread.join() + + # check the return code of the program + if Proc.returncode != 0: + if not isinstance(Command, type("")): + Command = " ".join(Command) + # print out the Response file and its content when make failure + RespFile = os.path.join(WorkingDir, 'OUTPUT', 'respfilelist.txt') + if os.path.isfile(RespFile): + f = open(RespFile) + RespContent = f.read() + f.close() + EdkLogger.info(RespContent) + + EdkLogger.error("build", COMMAND_FAILURE, ExtraData="%s [%s]" % (Command, WorkingDir)) + if ModuleAuto: + iau = IncludesAutoGen(WorkingDir,ModuleAuto) + if ModuleAuto.ToolChainFamily == TAB_COMPILER_MSFT: + iau.CreateDepsFileForMsvc(Proc.ProcOut) + else: + iau.UpdateDepsFileforNonMsvc() + iau.UpdateDepsFileforTrim() + iau.CreateModuleDeps() + iau.CreateDepsInclude() + iau.CreateDepsTarget() + return "%dms" % (int(round((time.time() - BeginTime) * 1000))) + +## The smallest unit that can be built in multi-thread build mode +# +# This is the base class of build unit. The "Obj" parameter must provide +# __str__(), __eq__() and __hash__() methods. Otherwise there could be build units +# missing build. +# +# Currently the "Obj" should be only ModuleAutoGen or PlatformAutoGen objects. +# +class BuildUnit: + ## The constructor + # + # @param self The object pointer + # @param Obj The object the build is working on + # @param Target The build target name, one of gSupportedTarget + # @param Dependency The BuildUnit(s) which must be completed in advance + # @param WorkingDir The directory build command starts in + # + def __init__(self, Obj, BuildCommand, Target, Dependency, WorkingDir="."): + self.BuildObject = Obj + self.Dependency = Dependency + self.WorkingDir = WorkingDir + self.Target = Target + self.BuildCommand = BuildCommand + if not BuildCommand: + EdkLogger.error("build", OPTION_MISSING, + "No build command found for this module. " + "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." % + (Obj.BuildTarget, Obj.ToolChain, Obj.Arch), + ExtraData=str(Obj)) + + + ## str() method + # + # It just returns the string representation of self.BuildObject + # + # @param self The object pointer + # + def __str__(self): + return str(self.BuildObject) + + ## "==" operator method + # + # It just compares self.BuildObject with "Other". So self.BuildObject must + # provide its own __eq__() method. + # + # @param self The object pointer + # @param Other The other BuildUnit object compared to + # + def __eq__(self, Other): + return Other and self.BuildObject == Other.BuildObject \ + and Other.BuildObject \ + and self.BuildObject.Arch == Other.BuildObject.Arch + + ## hash() method + # + # It just returns the hash value of self.BuildObject which must be hashable. + # + # @param self The object pointer + # + def __hash__(self): + return hash(self.BuildObject) + hash(self.BuildObject.Arch) + + def __repr__(self): + return repr(self.BuildObject) + +## The smallest module unit that can be built by nmake/make command in multi-thread build mode +# +# This class is for module build by nmake/make build system. The "Obj" parameter +# must provide __str__(), __eq__() and __hash__() methods. Otherwise there could +# be make units missing build. +# +# Currently the "Obj" should be only ModuleAutoGen object. +# +class ModuleMakeUnit(BuildUnit): + ## The constructor + # + # @param self The object pointer + # @param Obj The ModuleAutoGen object the build is working on + # @param Target The build target name, one of gSupportedTarget + # + def __init__(self, Obj, BuildCommand,Target): + Dependency = [ModuleMakeUnit(La, BuildCommand,Target) for La in Obj.LibraryAutoGenList] + BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir) + if Target in [None, "", "all"]: + self.Target = "tbuild" + +## The smallest platform unit that can be built by nmake/make command in multi-thread build mode +# +# This class is for platform build by nmake/make build system. The "Obj" parameter +# must provide __str__(), __eq__() and __hash__() methods. Otherwise there could +# be make units missing build. +# +# Currently the "Obj" should be only PlatformAutoGen object. +# +class PlatformMakeUnit(BuildUnit): + ## The constructor + # + # @param self The object pointer + # @param Obj The PlatformAutoGen object the build is working on + # @param Target The build target name, one of gSupportedTarget + # + def __init__(self, Obj, BuildCommand, Target): + Dependency = [ModuleMakeUnit(Lib, BuildCommand, Target) for Lib in self.BuildObject.LibraryAutoGenList] + Dependency.extend([ModuleMakeUnit(Mod, BuildCommand,Target) for Mod in self.BuildObject.ModuleAutoGenList]) + BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir) + +## The class representing the task of a module build or platform build +# +# This class manages the build tasks in multi-thread build mode. Its jobs include +# scheduling thread running, catching thread error, monitor the thread status, etc. +# +class BuildTask: + # queue for tasks waiting for schedule + _PendingQueue = OrderedDict() + _PendingQueueLock = threading.Lock() + + # queue for tasks ready for running + _ReadyQueue = OrderedDict() + _ReadyQueueLock = threading.Lock() + + # queue for run tasks + _RunningQueue = OrderedDict() + _RunningQueueLock = threading.Lock() + + # queue containing all build tasks, in case duplicate build + _TaskQueue = OrderedDict() + + # flag indicating error occurs in a running thread + _ErrorFlag = threading.Event() + _ErrorFlag.clear() + _ErrorMessage = "" + + # BoundedSemaphore object used to control the number of running threads + _Thread = None + + # flag indicating if the scheduler is started or not + _SchedulerStopped = threading.Event() + _SchedulerStopped.set() + + ## Start the task scheduler thread + # + # @param MaxThreadNumber The maximum thread number + # @param ExitFlag Flag used to end the scheduler + # + @staticmethod + def StartScheduler(MaxThreadNumber, ExitFlag): + SchedulerThread = Thread(target=BuildTask.Scheduler, args=(MaxThreadNumber, ExitFlag)) + SchedulerThread.setName("Build-Task-Scheduler") + SchedulerThread.setDaemon(False) + SchedulerThread.start() + # wait for the scheduler to be started, especially useful in Linux + while not BuildTask.IsOnGoing(): + time.sleep(0.01) + + ## Scheduler method + # + # @param MaxThreadNumber The maximum thread number + # @param ExitFlag Flag used to end the scheduler + # + @staticmethod + def Scheduler(MaxThreadNumber, ExitFlag): + BuildTask._SchedulerStopped.clear() + try: + # use BoundedSemaphore to control the maximum running threads + BuildTask._Thread = BoundedSemaphore(MaxThreadNumber) + # + # scheduling loop, which will exits when no pending/ready task and + # indicated to do so, or there's error in running thread + # + while (len(BuildTask._PendingQueue) > 0 or len(BuildTask._ReadyQueue) > 0 \ + or not ExitFlag.isSet()) and not BuildTask._ErrorFlag.isSet(): + EdkLogger.debug(EdkLogger.DEBUG_8, "Pending Queue (%d), Ready Queue (%d)" + % (len(BuildTask._PendingQueue), len(BuildTask._ReadyQueue))) + + # get all pending tasks + BuildTask._PendingQueueLock.acquire() + BuildObjectList = list(BuildTask._PendingQueue.keys()) + # + # check if their dependency is resolved, and if true, move them + # into ready queue + # + for BuildObject in BuildObjectList: + Bt = BuildTask._PendingQueue[BuildObject] + if Bt.IsReady(): + BuildTask._ReadyQueue[BuildObject] = BuildTask._PendingQueue.pop(BuildObject) + BuildTask._PendingQueueLock.release() + + # launch build thread until the maximum number of threads is reached + while not BuildTask._ErrorFlag.isSet(): + # empty ready queue, do nothing further + if len(BuildTask._ReadyQueue) == 0: + break + + # wait for active thread(s) exit + BuildTask._Thread.acquire(True) + + # start a new build thread + Bo, Bt = BuildTask._ReadyQueue.popitem() + + # move into running queue + BuildTask._RunningQueueLock.acquire() + BuildTask._RunningQueue[Bo] = Bt + BuildTask._RunningQueueLock.release() + + Bt.Start() + # avoid tense loop + time.sleep(0.01) + + # avoid tense loop + time.sleep(0.01) + + # wait for all running threads exit + if BuildTask._ErrorFlag.isSet(): + EdkLogger.quiet("\nWaiting for all build threads exit...") + # while not BuildTask._ErrorFlag.isSet() and \ + while len(BuildTask._RunningQueue) > 0: + EdkLogger.verbose("Waiting for thread ending...(%d)" % len(BuildTask._RunningQueue)) + EdkLogger.debug(EdkLogger.DEBUG_8, "Threads [%s]" % ", ".join(Th.getName() for Th in threading.enumerate())) + # avoid tense loop + time.sleep(0.1) + except BaseException as X: + # + # TRICK: hide the output of threads left running, so that the user can + # catch the error message easily + # + EdkLogger.SetLevel(EdkLogger.ERROR) + BuildTask._ErrorFlag.set() + BuildTask._ErrorMessage = "build thread scheduler error\n\t%s" % str(X) + + BuildTask._PendingQueue.clear() + BuildTask._ReadyQueue.clear() + BuildTask._RunningQueue.clear() + BuildTask._TaskQueue.clear() + BuildTask._SchedulerStopped.set() + + ## Wait for all running method exit + # + @staticmethod + def WaitForComplete(): + BuildTask._SchedulerStopped.wait() + + ## Check if the scheduler is running or not + # + @staticmethod + def IsOnGoing(): + return not BuildTask._SchedulerStopped.isSet() + + ## Abort the build + @staticmethod + def Abort(): + if BuildTask.IsOnGoing(): + BuildTask._ErrorFlag.set() + BuildTask.WaitForComplete() + + ## Check if there's error in running thread + # + # Since the main thread cannot catch exceptions in other thread, we have to + # use threading.Event to communicate this formation to main thread. + # + @staticmethod + def HasError(): + return BuildTask._ErrorFlag.isSet() + + ## Get error message in running thread + # + # Since the main thread cannot catch exceptions in other thread, we have to + # use a static variable to communicate this message to main thread. + # + @staticmethod + def GetErrorMessage(): + return BuildTask._ErrorMessage + + ## Factory method to create a BuildTask object + # + # This method will check if a module is building or has been built. And if + # true, just return the associated BuildTask object in the _TaskQueue. If + # not, create and return a new BuildTask object. The new BuildTask object + # will be appended to the _PendingQueue for scheduling later. + # + # @param BuildItem A BuildUnit object representing a build object + # @param Dependency The dependent build object of BuildItem + # + @staticmethod + def New(BuildItem, Dependency=None): + if BuildItem in BuildTask._TaskQueue: + Bt = BuildTask._TaskQueue[BuildItem] + return Bt + + Bt = BuildTask() + Bt._Init(BuildItem, Dependency) + BuildTask._TaskQueue[BuildItem] = Bt + + BuildTask._PendingQueueLock.acquire() + BuildTask._PendingQueue[BuildItem] = Bt + BuildTask._PendingQueueLock.release() + + return Bt + + ## The real constructor of BuildTask + # + # @param BuildItem A BuildUnit object representing a build object + # @param Dependency The dependent build object of BuildItem + # + def _Init(self, BuildItem, Dependency=None): + self.BuildItem = BuildItem + + self.DependencyList = [] + if Dependency is None: + Dependency = BuildItem.Dependency + else: + Dependency.extend(BuildItem.Dependency) + self.AddDependency(Dependency) + # flag indicating build completes, used to avoid unnecessary re-build + self.CompleteFlag = False + + ## Check if all dependent build tasks are completed or not + # + def IsReady(self): + ReadyFlag = True + for Dep in self.DependencyList: + if Dep.CompleteFlag == True: + continue + ReadyFlag = False + break + + return ReadyFlag + + ## Add dependent build task + # + # @param Dependency The list of dependent build objects + # + def AddDependency(self, Dependency): + for Dep in Dependency: + if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.CanSkipbyCache(GlobalData.gModuleCacheHit): + self.DependencyList.append(BuildTask.New(Dep)) # BuildTask list + + ## The thread wrapper of LaunchCommand function + # + # @param Command A list or string contains the call of the command + # @param WorkingDir The directory in which the program will be running + # + def _CommandThread(self, Command, WorkingDir): + try: + self.BuildItem.BuildObject.BuildTime = LaunchCommand(Command, WorkingDir,self.BuildItem.BuildObject) + self.CompleteFlag = True + + # Run hash operation post dependency to account for libs + # Run if --hash or --binary-destination + if GlobalData.gUseHashCache and not GlobalData.gBinCacheSource: + self.BuildItem.BuildObject.GenModuleHash() + if GlobalData.gBinCacheDest: + self.BuildItem.BuildObject.GenCMakeHash() + + except: + # + # TRICK: hide the output of threads left running, so that the user can + # catch the error message easily + # + if not BuildTask._ErrorFlag.isSet(): + GlobalData.gBuildingModule = "%s [%s, %s, %s]" % (str(self.BuildItem.BuildObject), + self.BuildItem.BuildObject.Arch, + self.BuildItem.BuildObject.ToolChain, + self.BuildItem.BuildObject.BuildTarget + ) + EdkLogger.SetLevel(EdkLogger.ERROR) + BuildTask._ErrorFlag.set() + BuildTask._ErrorMessage = "%s broken\n %s [%s]" % \ + (threading.currentThread().getName(), Command, WorkingDir) + + # indicate there's a thread is available for another build task + BuildTask._RunningQueueLock.acquire() + BuildTask._RunningQueue.pop(self.BuildItem) + BuildTask._RunningQueueLock.release() + BuildTask._Thread.release() + + ## Start build task thread + # + def Start(self): + EdkLogger.quiet("Building ... %s" % repr(self.BuildItem)) + Command = self.BuildItem.BuildCommand + [self.BuildItem.Target] + self.BuildTread = Thread(target=self._CommandThread, args=(Command, self.BuildItem.WorkingDir)) + self.BuildTread.setName("build thread") + self.BuildTread.setDaemon(False) + self.BuildTread.start() + +## The class contains the information related to EFI image +# +class PeImageInfo(): + ## Constructor + # + # Constructor will load all required image information. + # + # @param BaseName The full file path of image. + # @param Guid The GUID for image. + # @param Arch Arch of this image. + # @param OutputDir The output directory for image. + # @param DebugDir The debug directory for image. + # @param ImageClass PeImage Information + # + def __init__(self, BaseName, Guid, Arch, OutputDir, DebugDir, ImageClass): + self.BaseName = BaseName + self.Guid = Guid + self.Arch = Arch + self.OutputDir = OutputDir + self.DebugDir = DebugDir + self.Image = ImageClass + self.Image.Size = (self.Image.Size // 0x1000 + 1) * 0x1000 + +## The class implementing the EDK2 build process +# +# The build process includes: +# 1. Load configuration from target.txt and tools_def.txt in $(WORKSPACE)/Conf +# 2. Parse DSC file of active platform +# 3. Parse FDF file if any +# 4. Establish build database, including parse all other files (module, package) +# 5. Create AutoGen files (C code file, depex file, makefile) if necessary +# 6. Call build command +# +class Build(): + ## Constructor + # + # Constructor will load all necessary configurations, parse platform, modules + # and packages and the establish a database for AutoGen. + # + # @param Target The build command target, one of gSupportedTarget + # @param WorkspaceDir The directory of workspace + # @param BuildOptions Build options passed from command line + # + def __init__(self, Target, WorkspaceDir, BuildOptions,log_q): + self.WorkspaceDir = WorkspaceDir + self.Target = Target + self.PlatformFile = BuildOptions.PlatformFile + self.ModuleFile = BuildOptions.ModuleFile + self.ArchList = BuildOptions.TargetArch + self.ToolChainList = BuildOptions.ToolChain + self.BuildTargetList= BuildOptions.BuildTarget + self.Fdf = BuildOptions.FdfFile + self.FdList = BuildOptions.RomImage + self.FvList = BuildOptions.FvImage + self.CapList = BuildOptions.CapName + self.SilentMode = BuildOptions.SilentMode + self.ThreadNumber = 1 + self.SkipAutoGen = BuildOptions.SkipAutoGen + self.Reparse = BuildOptions.Reparse + self.SkuId = BuildOptions.SkuId + if self.SkuId: + GlobalData.gSKUID_CMD = self.SkuId + self.ConfDirectory = BuildOptions.ConfDirectory + self.SpawnMode = True + self.BuildReport = BuildReport(BuildOptions.ReportFile, BuildOptions.ReportType) + self.AutoGenTime = 0 + self.MakeTime = 0 + self.GenFdsTime = 0 + self.MakeFileName = "" + TargetObj = TargetTxtDict() + ToolDefObj = ToolDefDict((os.path.join(os.getenv("WORKSPACE"),"Conf"))) + self.TargetTxt = TargetObj.Target + self.ToolDef = ToolDefObj.ToolDef + GlobalData.BuildOptionPcd = BuildOptions.OptionPcd if BuildOptions.OptionPcd else [] + #Set global flag for build mode + GlobalData.gIgnoreSource = BuildOptions.IgnoreSources + GlobalData.gUseHashCache = BuildOptions.UseHashCache + GlobalData.gBinCacheDest = BuildOptions.BinCacheDest + GlobalData.gBinCacheSource = BuildOptions.BinCacheSource + GlobalData.gEnableGenfdsMultiThread = not BuildOptions.NoGenfdsMultiThread + GlobalData.gDisableIncludePathCheck = BuildOptions.DisableIncludePathCheck + + if GlobalData.gBinCacheDest and not GlobalData.gUseHashCache: + EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binary-destination must be used together with --hash.") + + if GlobalData.gBinCacheSource and not GlobalData.gUseHashCache: + EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binary-source must be used together with --hash.") + + if GlobalData.gBinCacheDest and GlobalData.gBinCacheSource: + EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binary-destination can not be used together with --binary-source.") + + if GlobalData.gBinCacheSource: + BinCacheSource = os.path.normpath(GlobalData.gBinCacheSource) + if not os.path.isabs(BinCacheSource): + BinCacheSource = mws.join(self.WorkspaceDir, BinCacheSource) + GlobalData.gBinCacheSource = BinCacheSource + else: + if GlobalData.gBinCacheSource is not None: + EdkLogger.error("build", OPTION_VALUE_INVALID, ExtraData="Invalid value of option --binary-source.") + + if GlobalData.gBinCacheDest: + BinCacheDest = os.path.normpath(GlobalData.gBinCacheDest) + if not os.path.isabs(BinCacheDest): + BinCacheDest = mws.join(self.WorkspaceDir, BinCacheDest) + GlobalData.gBinCacheDest = BinCacheDest + else: + if GlobalData.gBinCacheDest is not None: + EdkLogger.error("build", OPTION_VALUE_INVALID, ExtraData="Invalid value of option --binary-destination.") + + GlobalData.gDatabasePath = os.path.normpath(os.path.join(GlobalData.gConfDirectory, GlobalData.gDatabasePath)) + if not os.path.exists(os.path.join(GlobalData.gConfDirectory, '.cache')): + os.makedirs(os.path.join(GlobalData.gConfDirectory, '.cache')) + self.Db = BuildDB + self.BuildDatabase = self.Db.BuildObject + self.Platform = None + self.ToolChainFamily = None + self.LoadFixAddress = 0 + self.UniFlag = BuildOptions.Flag + self.BuildModules = [] + self.HashSkipModules = [] + self.Db_Flag = False + self.LaunchPrebuildFlag = False + self.PlatformBuildPath = os.path.join(GlobalData.gConfDirectory, '.cache', '.PlatformBuild') + if BuildOptions.CommandLength: + GlobalData.gCommandMaxLength = BuildOptions.CommandLength + + # print dot character during doing some time-consuming work + self.Progress = Utils.Progressor() + # print current build environment and configuration + EdkLogger.quiet("%-16s = %s" % ("WORKSPACE", os.environ["WORKSPACE"])) + if "PACKAGES_PATH" in os.environ: + # WORKSPACE env has been converted before. Print the same path style with WORKSPACE env. + EdkLogger.quiet("%-16s = %s" % ("PACKAGES_PATH", os.path.normcase(os.path.normpath(os.environ["PACKAGES_PATH"])))) + EdkLogger.quiet("%-16s = %s" % ("EDK_TOOLS_PATH", os.environ["EDK_TOOLS_PATH"])) + if "EDK_TOOLS_BIN" in os.environ: + # Print the same path style with WORKSPACE env. + EdkLogger.quiet("%-16s = %s" % ("EDK_TOOLS_BIN", os.path.normcase(os.path.normpath(os.environ["EDK_TOOLS_BIN"])))) + EdkLogger.quiet("%-16s = %s" % ("CONF_PATH", GlobalData.gConfDirectory)) + if "PYTHON3_ENABLE" in os.environ: + PYTHON3_ENABLE = os.environ["PYTHON3_ENABLE"] + if PYTHON3_ENABLE != "TRUE": + PYTHON3_ENABLE = "FALSE" + EdkLogger.quiet("%-16s = %s" % ("PYTHON3_ENABLE", PYTHON3_ENABLE)) + if "PYTHON_COMMAND" in os.environ: + EdkLogger.quiet("%-16s = %s" % ("PYTHON_COMMAND", os.environ["PYTHON_COMMAND"])) + self.InitPreBuild() + self.InitPostBuild() + if self.Prebuild: + EdkLogger.quiet("%-16s = %s" % ("PREBUILD", self.Prebuild)) + if self.Postbuild: + EdkLogger.quiet("%-16s = %s" % ("POSTBUILD", self.Postbuild)) + if self.Prebuild: + self.LaunchPrebuild() + TargetObj = TargetTxtDict() + ToolDefObj = ToolDefDict((os.path.join(os.getenv("WORKSPACE"), "Conf"))) + self.TargetTxt = TargetObj.Target + self.ToolDef = ToolDefObj.ToolDef + if not (self.LaunchPrebuildFlag and os.path.exists(self.PlatformBuildPath)): + self.InitBuild() + + self.AutoGenMgr = None + EdkLogger.info("") + os.chdir(self.WorkspaceDir) + self.log_q = log_q + GlobalData.file_lock = mp.Lock() + # Init cache data for local only + GlobalData.gPackageHashFile = dict() + GlobalData.gModulePreMakeCacheStatus = dict() + GlobalData.gModuleMakeCacheStatus = dict() + GlobalData.gHashChainStatus = dict() + GlobalData.gCMakeHashFile = dict() + GlobalData.gModuleHashFile = dict() + GlobalData.gFileHashDict = dict() + GlobalData.gModuleAllCacheStatus = set() + GlobalData.gModuleCacheHit = set() + + def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList,cqueue): + try: + if SkipAutoGen: + return True,0 + feedback_q = mp.Queue() + error_event = mp.Event() + FfsCmd = DataPipe.Get("FfsCommand") + if FfsCmd is None: + FfsCmd = {} + GlobalData.FfsCmd = FfsCmd + auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,GlobalData.file_lock,cqueue,self.log_q,error_event) for _ in range(self.ThreadNumber)] + self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q,error_event) + self.AutoGenMgr.start() + for w in auto_workers: + w.start() + if PcdMaList is not None: + for PcdMa in PcdMaList: + # SourceFileList calling sequence impact the makefile string sequence. + # Create cached SourceFileList here to unify its calling sequence for both + # CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile. + RetVal = PcdMa.SourceFileList + # Force cache miss for PCD driver + if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and self.Target in [None, "", "all"]: + cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, "PreMakeCache", False)) + + PcdMa.CreateCodeFile(False) + PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.Path, PcdMa.Arch),[])) + PcdMa.CreateAsBuiltInf() + # Force cache miss for PCD driver + if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: + cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, "MakeCache", False)) + + self.AutoGenMgr.join() + rt = self.AutoGenMgr.Status + err = 0 + if not rt: + err = UNKNOWN_ERROR + return rt, err + except FatalError as e: + return False, e.args[0] + except: + return False, UNKNOWN_ERROR + + ## Add TOOLCHAIN and FAMILY declared in DSC [BuildOptions] to ToolsDefTxtDatabase. + # + # Loop through the set of build targets, tool chains, and archs provided on either + # the command line or in target.txt to discover FAMILY and TOOLCHAIN delclarations + # in [BuildOptions] sections that may be within !if expressions that may use + # $(TARGET), $(TOOLCHAIN), $(TOOLCHAIN_TAG), or $(ARCH) operands. + # + def GetToolChainAndFamilyFromDsc (self, File): + SavedGlobalDefines = GlobalData.gGlobalDefines.copy() + for BuildTarget in self.BuildTargetList: + GlobalData.gGlobalDefines['TARGET'] = BuildTarget + for BuildToolChain in self.ToolChainList: + GlobalData.gGlobalDefines['TOOLCHAIN'] = BuildToolChain + GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = BuildToolChain + for BuildArch in self.ArchList: + GlobalData.gGlobalDefines['ARCH'] = BuildArch + dscobj = self.BuildDatabase[File, BuildArch] + for KeyFamily, Key, KeyCodeBase in dscobj.BuildOptions: + try: + Target, ToolChain, Arch, Tool, Attr = Key.split('_') + except: + continue + if ToolChain == TAB_STAR or Attr != TAB_TOD_DEFINES_FAMILY: + continue + try: + Family = dscobj.BuildOptions[(KeyFamily, Key, KeyCodeBase)] + Family = Family.strip().strip('=').strip() + except: + continue + if TAB_TOD_DEFINES_FAMILY not in self.ToolDef.ToolsDefTxtDatabase: + self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY] = {} + if ToolChain not in self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY]: + self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY][ToolChain] = Family + if TAB_TOD_DEFINES_BUILDRULEFAMILY not in self.ToolDef.ToolsDefTxtDatabase: + self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY] = {} + if ToolChain not in self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY]: + self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY][ToolChain] = Family + if TAB_TOD_DEFINES_TOOL_CHAIN_TAG not in self.ToolDef.ToolsDefTxtDatabase: + self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG] = [] + if ToolChain not in self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG]: + self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG].append(ToolChain) + GlobalData.gGlobalDefines = SavedGlobalDefines + + ## Load configuration + # + # This method will parse target.txt and get the build configurations. + # + def LoadConfiguration(self): + + # if no ARCH given in command line, get it from target.txt + if not self.ArchList: + self.ArchList = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TARGET_ARCH] + self.ArchList = tuple(self.ArchList) + + # if no build target given in command line, get it from target.txt + if not self.BuildTargetList: + self.BuildTargetList = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TARGET] + + # if no tool chain given in command line, get it from target.txt + if not self.ToolChainList: + self.ToolChainList = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TOOL_CHAIN_TAG] + if self.ToolChainList is None or len(self.ToolChainList) == 0: + EdkLogger.error("build", RESOURCE_NOT_AVAILABLE, ExtraData="No toolchain given. Don't know how to build.\n") + + if not self.PlatformFile: + PlatformFile = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_ACTIVE_PLATFORM] + if not PlatformFile: + # Try to find one in current directory + WorkingDirectory = os.getcwd() + FileList = glob.glob(os.path.normpath(os.path.join(WorkingDirectory, '*.dsc'))) + FileNum = len(FileList) + if FileNum >= 2: + EdkLogger.error("build", OPTION_MISSING, + ExtraData="There are %d DSC files in %s. Use '-p' to specify one.\n" % (FileNum, WorkingDirectory)) + elif FileNum == 1: + PlatformFile = FileList[0] + else: + EdkLogger.error("build", RESOURCE_NOT_AVAILABLE, + ExtraData="No active platform specified in target.txt or command line! Nothing can be built.\n") + + self.PlatformFile = PathClass(NormFile(PlatformFile, self.WorkspaceDir), self.WorkspaceDir) + + self.GetToolChainAndFamilyFromDsc (self.PlatformFile) + + # check if the tool chains are defined or not + NewToolChainList = [] + for ToolChain in self.ToolChainList: + if ToolChain not in self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG]: + EdkLogger.warn("build", "Tool chain [%s] is not defined" % ToolChain) + else: + NewToolChainList.append(ToolChain) + # if no tool chain available, break the build + if len(NewToolChainList) == 0: + EdkLogger.error("build", RESOURCE_NOT_AVAILABLE, + ExtraData="[%s] not defined. No toolchain available for build!\n" % ", ".join(self.ToolChainList)) + else: + self.ToolChainList = NewToolChainList + + ToolChainFamily = [] + ToolDefinition = self.ToolDef.ToolsDefTxtDatabase + for Tool in self.ToolChainList: + if TAB_TOD_DEFINES_FAMILY not in ToolDefinition or Tool not in ToolDefinition[TAB_TOD_DEFINES_FAMILY] \ + or not ToolDefinition[TAB_TOD_DEFINES_FAMILY][Tool]: + EdkLogger.warn("build", "No tool chain family found in configuration for %s. Default to MSFT." % Tool) + ToolChainFamily.append(TAB_COMPILER_MSFT) + else: + ToolChainFamily.append(ToolDefinition[TAB_TOD_DEFINES_FAMILY][Tool]) + self.ToolChainFamily = ToolChainFamily + + self.ThreadNumber = ThreadNum() + ## Initialize build configuration + # + # This method will parse DSC file and merge the configurations from + # command line and target.txt, then get the final build configurations. + # + def InitBuild(self): + # parse target.txt, tools_def.txt, and platform file + self.LoadConfiguration() + + # Allow case-insensitive for those from command line or configuration file + ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc", False) + if ErrorCode != 0: + EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo) + + + def InitPreBuild(self): + self.LoadConfiguration() + ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc", False) + if ErrorCode != 0: + EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo) + if self.BuildTargetList: + GlobalData.gGlobalDefines['TARGET'] = self.BuildTargetList[0] + if self.ArchList: + GlobalData.gGlobalDefines['ARCH'] = self.ArchList[0] + if self.ToolChainList: + GlobalData.gGlobalDefines['TOOLCHAIN'] = self.ToolChainList[0] + GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = self.ToolChainList[0] + if self.ToolChainFamily: + GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[0] + if 'PREBUILD' in GlobalData.gCommandLineDefines: + self.Prebuild = GlobalData.gCommandLineDefines.get('PREBUILD') + else: + self.Db_Flag = True + Platform = self.Db.MapPlatform(str(self.PlatformFile)) + self.Prebuild = str(Platform.Prebuild) + if self.Prebuild: + PrebuildList = [] + # + # Evaluate all arguments and convert arguments that are WORKSPACE + # relative paths to absolute paths. Filter arguments that look like + # flags or do not follow the file/dir naming rules to avoid false + # positives on this conversion. + # + for Arg in self.Prebuild.split(): + # + # Do not modify Arg if it looks like a flag or an absolute file path + # + if Arg.startswith('-') or os.path.isabs(Arg): + PrebuildList.append(Arg) + continue + # + # Do not modify Arg if it does not look like a Workspace relative + # path that starts with a valid package directory name + # + if not Arg[0].isalpha() or os.path.dirname(Arg) == '': + PrebuildList.append(Arg) + continue + # + # If Arg looks like a WORKSPACE relative path, then convert to an + # absolute path and check to see if the file exists. + # + Temp = mws.join(self.WorkspaceDir, Arg) + if os.path.isfile(Temp): + Arg = Temp + PrebuildList.append(Arg) + self.Prebuild = ' '.join(PrebuildList) + self.Prebuild += self.PassCommandOption(self.BuildTargetList, self.ArchList, self.ToolChainList, self.PlatformFile, self.Target) + + def InitPostBuild(self): + if 'POSTBUILD' in GlobalData.gCommandLineDefines: + self.Postbuild = GlobalData.gCommandLineDefines.get('POSTBUILD') + else: + Platform = self.Db.MapPlatform(str(self.PlatformFile)) + self.Postbuild = str(Platform.Postbuild) + if self.Postbuild: + PostbuildList = [] + # + # Evaluate all arguments and convert arguments that are WORKSPACE + # relative paths to absolute paths. Filter arguments that look like + # flags or do not follow the file/dir naming rules to avoid false + # positives on this conversion. + # + for Arg in self.Postbuild.split(): + # + # Do not modify Arg if it looks like a flag or an absolute file path + # + if Arg.startswith('-') or os.path.isabs(Arg): + PostbuildList.append(Arg) + continue + # + # Do not modify Arg if it does not look like a Workspace relative + # path that starts with a valid package directory name + # + if not Arg[0].isalpha() or os.path.dirname(Arg) == '': + PostbuildList.append(Arg) + continue + # + # If Arg looks like a WORKSPACE relative path, then convert to an + # absolute path and check to see if the file exists. + # + Temp = mws.join(self.WorkspaceDir, Arg) + if os.path.isfile(Temp): + Arg = Temp + PostbuildList.append(Arg) + self.Postbuild = ' '.join(PostbuildList) + self.Postbuild += self.PassCommandOption(self.BuildTargetList, self.ArchList, self.ToolChainList, self.PlatformFile, self.Target) + + def PassCommandOption(self, BuildTarget, TargetArch, ToolChain, PlatformFile, Target): + BuildStr = '' + if GlobalData.gCommand and isinstance(GlobalData.gCommand, list): + BuildStr += ' ' + ' '.join(GlobalData.gCommand) + TargetFlag = False + ArchFlag = False + ToolChainFlag = False + PlatformFileFlag = False + + if GlobalData.gOptions and not GlobalData.gOptions.BuildTarget: + TargetFlag = True + if GlobalData.gOptions and not GlobalData.gOptions.TargetArch: + ArchFlag = True + if GlobalData.gOptions and not GlobalData.gOptions.ToolChain: + ToolChainFlag = True + if GlobalData.gOptions and not GlobalData.gOptions.PlatformFile: + PlatformFileFlag = True + + if TargetFlag and BuildTarget: + if isinstance(BuildTarget, list) or isinstance(BuildTarget, tuple): + BuildStr += ' -b ' + ' -b '.join(BuildTarget) + elif isinstance(BuildTarget, str): + BuildStr += ' -b ' + BuildTarget + if ArchFlag and TargetArch: + if isinstance(TargetArch, list) or isinstance(TargetArch, tuple): + BuildStr += ' -a ' + ' -a '.join(TargetArch) + elif isinstance(TargetArch, str): + BuildStr += ' -a ' + TargetArch + if ToolChainFlag and ToolChain: + if isinstance(ToolChain, list) or isinstance(ToolChain, tuple): + BuildStr += ' -t ' + ' -t '.join(ToolChain) + elif isinstance(ToolChain, str): + BuildStr += ' -t ' + ToolChain + if PlatformFileFlag and PlatformFile: + if isinstance(PlatformFile, list) or isinstance(PlatformFile, tuple): + BuildStr += ' -p ' + ' -p '.join(PlatformFile) + elif isinstance(PlatformFile, str): + BuildStr += ' -p' + PlatformFile + BuildStr += ' --conf=' + GlobalData.gConfDirectory + if Target: + BuildStr += ' ' + Target + + return BuildStr + + def LaunchPrebuild(self): + if self.Prebuild: + EdkLogger.info("\n- Prebuild Start -\n") + self.LaunchPrebuildFlag = True + # + # The purpose of .PrebuildEnv file is capture environment variable settings set by the prebuild script + # and preserve them for the rest of the main build step, because the child process environment will + # evaporate as soon as it exits, we cannot get it in build step. + # + PrebuildEnvFile = os.path.join(GlobalData.gConfDirectory, '.cache', '.PrebuildEnv') + if os.path.isfile(PrebuildEnvFile): + os.remove(PrebuildEnvFile) + if os.path.isfile(self.PlatformBuildPath): + os.remove(self.PlatformBuildPath) + if sys.platform == "win32": + args = ' && '.join((self.Prebuild, 'set > ' + PrebuildEnvFile)) + Process = Popen(args, stdout=PIPE, stderr=PIPE, shell=True) + else: + args = ' && '.join((self.Prebuild, 'env > ' + PrebuildEnvFile)) + Process = Popen(args, stdout=PIPE, stderr=PIPE, shell=True) + + # launch two threads to read the STDOUT and STDERR + EndOfProcedure = Event() + EndOfProcedure.clear() + if Process.stdout: + StdOutThread = Thread(target=ReadMessage, args=(Process.stdout, EdkLogger.info, EndOfProcedure)) + StdOutThread.setName("STDOUT-Redirector") + StdOutThread.setDaemon(False) + StdOutThread.start() + + if Process.stderr: + StdErrThread = Thread(target=ReadMessage, args=(Process.stderr, EdkLogger.quiet, EndOfProcedure)) + StdErrThread.setName("STDERR-Redirector") + StdErrThread.setDaemon(False) + StdErrThread.start() + # waiting for program exit + Process.wait() + + if Process.stdout: + StdOutThread.join() + if Process.stderr: + StdErrThread.join() + if Process.returncode != 0 : + EdkLogger.error("Prebuild", PREBUILD_ERROR, 'Prebuild process is not success!') + + if os.path.exists(PrebuildEnvFile): + f = open(PrebuildEnvFile) + envs = f.readlines() + f.close() + envs = [l.split("=", 1) for l in envs ] + envs = [[I.strip() for I in item] for item in envs if len(item) == 2] + os.environ.update(dict(envs)) + EdkLogger.info("\n- Prebuild Done -\n") + + def LaunchPostbuild(self): + if self.Postbuild: + EdkLogger.info("\n- Postbuild Start -\n") + if sys.platform == "win32": + Process = Popen(self.Postbuild, stdout=PIPE, stderr=PIPE, shell=True) + else: + Process = Popen(self.Postbuild, stdout=PIPE, stderr=PIPE, shell=True) + # launch two threads to read the STDOUT and STDERR + EndOfProcedure = Event() + EndOfProcedure.clear() + if Process.stdout: + StdOutThread = Thread(target=ReadMessage, args=(Process.stdout, EdkLogger.info, EndOfProcedure)) + StdOutThread.setName("STDOUT-Redirector") + StdOutThread.setDaemon(False) + StdOutThread.start() + + if Process.stderr: + StdErrThread = Thread(target=ReadMessage, args=(Process.stderr, EdkLogger.quiet, EndOfProcedure)) + StdErrThread.setName("STDERR-Redirector") + StdErrThread.setDaemon(False) + StdErrThread.start() + # waiting for program exit + Process.wait() + + if Process.stdout: + StdOutThread.join() + if Process.stderr: + StdErrThread.join() + if Process.returncode != 0 : + EdkLogger.error("Postbuild", POSTBUILD_ERROR, 'Postbuild process is not success!') + EdkLogger.info("\n- Postbuild Done -\n") + + ## Build a module or platform + # + # Create autogen code and makefile for a module or platform, and the launch + # "make" command to build it + # + # @param Target The target of build command + # @param Platform The platform file + # @param Module The module file + # @param BuildTarget The name of build target, one of "DEBUG", "RELEASE" + # @param ToolChain The name of toolchain to build + # @param Arch The arch of the module/platform + # @param CreateDepModuleCodeFile Flag used to indicate creating code + # for dependent modules/Libraries + # @param CreateDepModuleMakeFile Flag used to indicate creating makefile + # for dependent modules/Libraries + # + def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand=None, PcdMaList=None): + if AutoGenObject is None: + return False + if FfsCommand is None: + FfsCommand = {} + # skip file generation for cleanxxx targets, run and fds target + if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: + # for target which must generate AutoGen code and makefile + mqueue = mp.Queue() + for m in AutoGenObject.GetAllModuleInfo: + mqueue.put(m) + mqueue.put((None,None,None,None,None,None,None)) + AutoGenObject.DataPipe.DataContainer = {"CommandTarget": self.Target} + AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp} + AutoGenObject.CreateLibModuelDirs() + AutoGenObject.DataPipe.DataContainer = {"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList} + AutoGenObject.DataPipe.DataContainer = {"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList} + AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict": AutoGenObject.Workspace.GenFdsCommandDict} + self.Progress.Start("Generating makefile and code") + data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch)) + AutoGenObject.DataPipe.dump(data_pipe_file) + cqueue = mp.Queue() + autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, cqueue) + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") + with open(AutoGenIdFile,"w") as fw: + fw.write("Arch=%s\n" % "|".join((AutoGenObject.Workspace.ArchList))) + fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir) + fw.write("PlatformGuid=%s\n" % str(AutoGenObject.Guid)) + self.Progress.Stop("done!") + if not autogen_rt: + self.AutoGenMgr.TerminateWorkers() + self.AutoGenMgr.join(1) + raise FatalError(errorcode) + AutoGenObject.CreateCodeFile(False) + AutoGenObject.CreateMakeFile(False) + else: + # always recreate top/platform makefile when clean, just in case of inconsistency + AutoGenObject.CreateCodeFile(True) + AutoGenObject.CreateMakeFile(True) + + if EdkLogger.GetLevel() == EdkLogger.QUIET: + EdkLogger.quiet("Building ... %s" % repr(AutoGenObject)) + + BuildCommand = AutoGenObject.BuildCommand + if BuildCommand is None or len(BuildCommand) == 0: + EdkLogger.error("build", OPTION_MISSING, + "No build command found for this module. " + "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." % + (AutoGenObject.BuildTarget, AutoGenObject.ToolChain, AutoGenObject.Arch), + ExtraData=str(AutoGenObject)) + + # run + if Target == 'run': + return True + + # build modules + if BuildModule: + BuildCommand = BuildCommand + [Target] + LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir) + if GlobalData.gBinCacheDest: + self.GenDestCache() + elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource: + # Only for --hash + # Update PreMakeCacheChain files + self.GenLocalPreMakeCache() + self.BuildModules = [] + return True + + # build library + if Target == 'libraries': + DirList = [] + for Lib in AutoGenObject.LibraryAutoGenList: + if not Lib.IsBinaryModule: + DirList.append((os.path.join(AutoGenObject.BuildDir, Lib.BuildDir),Lib)) + for Lib, LibAutoGen in DirList: + NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, self.MakeFileName)), 'pbuild'] + LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,LibAutoGen) + return True + + # build module + if Target == 'modules': + DirList = [] + for Lib in AutoGenObject.LibraryAutoGenList: + if not Lib.IsBinaryModule: + DirList.append((os.path.join(AutoGenObject.BuildDir, Lib.BuildDir),Lib)) + for Lib, LibAutoGen in DirList: + NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, self.MakeFileName)), 'pbuild'] + LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,LibAutoGen) + + DirList = [] + for ModuleAutoGen in AutoGenObject.ModuleAutoGenList: + if not ModuleAutoGen.IsBinaryModule: + DirList.append((os.path.join(AutoGenObject.BuildDir, ModuleAutoGen.BuildDir),ModuleAutoGen)) + for Mod,ModAutoGen in DirList: + NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Mod, self.MakeFileName)), 'pbuild'] + LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,ModAutoGen) + self.CreateAsBuiltInf() + if GlobalData.gBinCacheDest: + self.GenDestCache() + elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource: + # Only for --hash + # Update PreMakeCacheChain files + self.GenLocalPreMakeCache() + self.BuildModules = [] + return True + + # cleanlib + if Target == 'cleanlib': + for Lib in AutoGenObject.LibraryBuildDirectoryList: + LibMakefile = os.path.normpath(os.path.join(Lib, self.MakeFileName)) + if os.path.exists(LibMakefile): + NewBuildCommand = BuildCommand + ['-f', LibMakefile, 'cleanall'] + LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir) + return True + + # clean + if Target == 'clean': + for Mod in AutoGenObject.ModuleBuildDirectoryList: + ModMakefile = os.path.normpath(os.path.join(Mod, self.MakeFileName)) + if os.path.exists(ModMakefile): + NewBuildCommand = BuildCommand + ['-f', ModMakefile, 'cleanall'] + LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir) + for Lib in AutoGenObject.LibraryBuildDirectoryList: + LibMakefile = os.path.normpath(os.path.join(Lib, self.MakeFileName)) + if os.path.exists(LibMakefile): + NewBuildCommand = BuildCommand + ['-f', LibMakefile, 'cleanall'] + LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir) + return True + + # cleanall + if Target == 'cleanall': + try: + #os.rmdir(AutoGenObject.BuildDir) + RemoveDirectory(AutoGenObject.BuildDir, True) + except WindowsError as X: + EdkLogger.error("build", FILE_DELETE_FAILURE, ExtraData=str(X)) + return True + + ## Build a module or platform + # + # Create autogen code and makefile for a module or platform, and the launch + # "make" command to build it + # + # @param Target The target of build command + # @param Platform The platform file + # @param Module The module file + # @param BuildTarget The name of build target, one of "DEBUG", "RELEASE" + # @param ToolChain The name of toolchain to build + # @param Arch The arch of the module/platform + # @param CreateDepModuleCodeFile Flag used to indicate creating code + # for dependent modules/Libraries + # @param CreateDepModuleMakeFile Flag used to indicate creating makefile + # for dependent modules/Libraries + # + def _Build(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False): + if AutoGenObject is None: + return False + + # skip file generation for cleanxxx targets, run and fds target + if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: + # for target which must generate AutoGen code and makefile + if not self.SkipAutoGen or Target == 'genc': + self.Progress.Start("Generating code") + AutoGenObject.CreateCodeFile(CreateDepsCodeFile) + self.Progress.Stop("done!") + if Target == "genc": + return True + + if not self.SkipAutoGen or Target == 'genmake': + self.Progress.Start("Generating makefile") + AutoGenObject.CreateMakeFile(CreateDepsMakeFile) + #AutoGenObject.CreateAsBuiltInf() + self.Progress.Stop("done!") + if Target == "genmake": + return True + else: + # always recreate top/platform makefile when clean, just in case of inconsistency + AutoGenObject.CreateCodeFile(True) + AutoGenObject.CreateMakeFile(True) + + if EdkLogger.GetLevel() == EdkLogger.QUIET: + EdkLogger.quiet("Building ... %s" % repr(AutoGenObject)) + + BuildCommand = AutoGenObject.BuildCommand + if BuildCommand is None or len(BuildCommand) == 0: + EdkLogger.error("build", OPTION_MISSING, + "No build command found for this module. " + "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." % + (AutoGenObject.BuildTarget, AutoGenObject.ToolChain, AutoGenObject.Arch), + ExtraData=str(AutoGenObject)) + + # build modules + if BuildModule: + if Target != 'fds': + BuildCommand = BuildCommand + [Target] + AutoGenObject.BuildTime = LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir) + self.CreateAsBuiltInf() + if GlobalData.gBinCacheDest: + self.GenDestCache() + elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource: + # Only for --hash + # Update PreMakeCacheChain files + self.GenLocalPreMakeCache() + self.BuildModules = [] + return True + + # genfds + if Target == 'fds': + if GenFdsApi(AutoGenObject.GenFdsCommandDict, self.Db): + EdkLogger.error("build", COMMAND_FAILURE) + Threshold = self.GetFreeSizeThreshold() + if Threshold: + self.CheckFreeSizeThreshold(Threshold, AutoGenObject.FvDir) + return True + + # run + if Target == 'run': + return True + + # build library + if Target == 'libraries': + pass + + # not build modules + + + # cleanall + if Target == 'cleanall': + try: + #os.rmdir(AutoGenObject.BuildDir) + RemoveDirectory(AutoGenObject.BuildDir, True) + except WindowsError as X: + EdkLogger.error("build", FILE_DELETE_FAILURE, ExtraData=str(X)) + return True + + ## Rebase module image and Get function address for the input module list. + # + def _RebaseModule (self, MapBuffer, BaseAddress, ModuleList, AddrIsOffset = True, ModeIsSmm = False): + if ModeIsSmm: + AddrIsOffset = False + for InfFile in ModuleList: + sys.stdout.write (".") + sys.stdout.flush() + ModuleInfo = ModuleList[InfFile] + ModuleName = ModuleInfo.BaseName + ModuleOutputImage = ModuleInfo.Image.FileName + ModuleDebugImage = os.path.join(ModuleInfo.DebugDir, ModuleInfo.BaseName + '.efi') + ## for SMM module in SMRAM, the SMRAM will be allocated from base to top. + if not ModeIsSmm: + BaseAddress = BaseAddress - ModuleInfo.Image.Size + # + # Update Image to new BaseAddress by GenFw tool + # + LaunchCommand(["GenFw", "--rebase", str(BaseAddress), "-r", ModuleOutputImage], ModuleInfo.OutputDir) + LaunchCommand(["GenFw", "--rebase", str(BaseAddress), "-r", ModuleDebugImage], ModuleInfo.DebugDir) + else: + # + # Set new address to the section header only for SMM driver. + # + LaunchCommand(["GenFw", "--address", str(BaseAddress), "-r", ModuleOutputImage], ModuleInfo.OutputDir) + LaunchCommand(["GenFw", "--address", str(BaseAddress), "-r", ModuleDebugImage], ModuleInfo.DebugDir) + # + # Collect function address from Map file + # + ImageMapTable = ModuleOutputImage.replace('.efi', '.map') + FunctionList = [] + if os.path.exists(ImageMapTable): + OrigImageBaseAddress = 0 + ImageMap = open(ImageMapTable, 'r') + for LinStr in ImageMap: + if len (LinStr.strip()) == 0: + continue + # + # Get the preferred address set on link time. + # + if LinStr.find ('Preferred load address is') != -1: + StrList = LinStr.split() + OrigImageBaseAddress = int (StrList[len(StrList) - 1], 16) + + StrList = LinStr.split() + if len (StrList) > 4: + if StrList[3] == 'f' or StrList[3] == 'F': + Name = StrList[1] + RelativeAddress = int (StrList[2], 16) - OrigImageBaseAddress + FunctionList.append ((Name, RelativeAddress)) + + ImageMap.close() + # + # Add general information. + # + if ModeIsSmm: + MapBuffer.append('\n\n%s (Fixed SMRAM Offset, BaseAddress=0x%010X, EntryPoint=0x%010X)\n' % (ModuleName, BaseAddress, BaseAddress + ModuleInfo.Image.EntryPoint)) + elif AddrIsOffset: + MapBuffer.append('\n\n%s (Fixed Memory Offset, BaseAddress=-0x%010X, EntryPoint=-0x%010X)\n' % (ModuleName, 0 - BaseAddress, 0 - (BaseAddress + ModuleInfo.Image.EntryPoint))) + else: + MapBuffer.append('\n\n%s (Fixed Memory Address, BaseAddress=0x%010X, EntryPoint=0x%010X)\n' % (ModuleName, BaseAddress, BaseAddress + ModuleInfo.Image.EntryPoint)) + # + # Add guid and general seciton section. + # + TextSectionAddress = 0 + DataSectionAddress = 0 + for SectionHeader in ModuleInfo.Image.SectionHeaderList: + if SectionHeader[0] == '.text': + TextSectionAddress = SectionHeader[1] + elif SectionHeader[0] in ['.data', '.sdata']: + DataSectionAddress = SectionHeader[1] + if AddrIsOffset: + MapBuffer.append('(GUID=%s, .textbaseaddress=-0x%010X, .databaseaddress=-0x%010X)\n' % (ModuleInfo.Guid, 0 - (BaseAddress + TextSectionAddress), 0 - (BaseAddress + DataSectionAddress))) + else: + MapBuffer.append('(GUID=%s, .textbaseaddress=0x%010X, .databaseaddress=0x%010X)\n' % (ModuleInfo.Guid, BaseAddress + TextSectionAddress, BaseAddress + DataSectionAddress)) + # + # Add debug image full path. + # + MapBuffer.append('(IMAGE=%s)\n\n' % (ModuleDebugImage)) + # + # Add function address + # + for Function in FunctionList: + if AddrIsOffset: + MapBuffer.append(' -0x%010X %s\n' % (0 - (BaseAddress + Function[1]), Function[0])) + else: + MapBuffer.append(' 0x%010X %s\n' % (BaseAddress + Function[1], Function[0])) + ImageMap.close() + + # + # for SMM module in SMRAM, the SMRAM will be allocated from base to top. + # + if ModeIsSmm: + BaseAddress = BaseAddress + ModuleInfo.Image.Size + + ## Collect MAP information of all FVs + # + def _CollectFvMapBuffer (self, MapBuffer, Wa, ModuleList): + if self.Fdf: + # First get the XIP base address for FV map file. + GuidPattern = re.compile("[-a-fA-F0-9]+") + GuidName = re.compile(r"\(GUID=[-a-fA-F0-9]+") + for FvName in Wa.FdfProfile.FvDict: + FvMapBuffer = os.path.join(Wa.FvDir, FvName + '.Fv.map') + if not os.path.exists(FvMapBuffer): + continue + FvMap = open(FvMapBuffer, 'r') + #skip FV size information + FvMap.readline() + FvMap.readline() + FvMap.readline() + FvMap.readline() + for Line in FvMap: + MatchGuid = GuidPattern.match(Line) + if MatchGuid is not None: + # + # Replace GUID with module name + # + GuidString = MatchGuid.group() + if GuidString.upper() in ModuleList: + Line = Line.replace(GuidString, ModuleList[GuidString.upper()].Name) + MapBuffer.append(Line) + # + # Add the debug image full path. + # + MatchGuid = GuidName.match(Line) + if MatchGuid is not None: + GuidString = MatchGuid.group().split("=")[1] + if GuidString.upper() in ModuleList: + MapBuffer.append('(IMAGE=%s)\n' % (os.path.join(ModuleList[GuidString.upper()].DebugDir, ModuleList[GuidString.upper()].Name + '.efi'))) + + FvMap.close() + + ## Collect MAP information of all modules + # + def _CollectModuleMapBuffer (self, MapBuffer, ModuleList): + sys.stdout.write ("Generate Load Module At Fix Address Map") + sys.stdout.flush() + PatchEfiImageList = [] + PeiModuleList = {} + BtModuleList = {} + RtModuleList = {} + SmmModuleList = {} + PeiSize = 0 + BtSize = 0 + RtSize = 0 + # reserve 4K size in SMRAM to make SMM module address not from 0. + SmmSize = 0x1000 + for ModuleGuid in ModuleList: + Module = ModuleList[ModuleGuid] + GlobalData.gProcessingFile = "%s [%s, %s, %s]" % (Module.MetaFile, Module.Arch, Module.ToolChain, Module.BuildTarget) + + OutputImageFile = '' + for ResultFile in Module.CodaTargetList: + if str(ResultFile.Target).endswith('.efi'): + # + # module list for PEI, DXE, RUNTIME and SMM + # + OutputImageFile = os.path.join(Module.OutputDir, Module.Name + '.efi') + ImageClass = PeImageClass (OutputImageFile) + if not ImageClass.IsValid: + EdkLogger.error("build", FILE_PARSE_FAILURE, ExtraData=ImageClass.ErrorInfo) + ImageInfo = PeImageInfo(Module.Name, Module.Guid, Module.Arch, Module.OutputDir, Module.DebugDir, ImageClass) + if Module.ModuleType in [SUP_MODULE_PEI_CORE, SUP_MODULE_PEIM, EDK_COMPONENT_TYPE_COMBINED_PEIM_DRIVER, EDK_COMPONENT_TYPE_PIC_PEIM, EDK_COMPONENT_TYPE_RELOCATABLE_PEIM, SUP_MODULE_DXE_CORE]: + PeiModuleList[Module.MetaFile] = ImageInfo + PeiSize += ImageInfo.Image.Size + elif Module.ModuleType in [EDK_COMPONENT_TYPE_BS_DRIVER, SUP_MODULE_DXE_DRIVER, SUP_MODULE_UEFI_DRIVER]: + BtModuleList[Module.MetaFile] = ImageInfo + BtSize += ImageInfo.Image.Size + elif Module.ModuleType in [SUP_MODULE_DXE_RUNTIME_DRIVER, EDK_COMPONENT_TYPE_RT_DRIVER, SUP_MODULE_DXE_SAL_DRIVER, EDK_COMPONENT_TYPE_SAL_RT_DRIVER]: + RtModuleList[Module.MetaFile] = ImageInfo + RtSize += ImageInfo.Image.Size + elif Module.ModuleType in [SUP_MODULE_SMM_CORE, SUP_MODULE_DXE_SMM_DRIVER, SUP_MODULE_MM_STANDALONE, SUP_MODULE_MM_CORE_STANDALONE]: + SmmModuleList[Module.MetaFile] = ImageInfo + SmmSize += ImageInfo.Image.Size + if Module.ModuleType == SUP_MODULE_DXE_SMM_DRIVER: + PiSpecVersion = Module.Module.Specification.get('PI_SPECIFICATION_VERSION', '0x00000000') + # for PI specification < PI1.1, DXE_SMM_DRIVER also runs as BOOT time driver. + if int(PiSpecVersion, 16) < 0x0001000A: + BtModuleList[Module.MetaFile] = ImageInfo + BtSize += ImageInfo.Image.Size + break + # + # EFI image is final target. + # Check EFI image contains patchable FixAddress related PCDs. + # + if OutputImageFile != '': + ModuleIsPatch = False + for Pcd in Module.ModulePcdList: + if Pcd.Type == TAB_PCDS_PATCHABLE_IN_MODULE and Pcd.TokenCName in TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SET: + ModuleIsPatch = True + break + if not ModuleIsPatch: + for Pcd in Module.LibraryPcdList: + if Pcd.Type == TAB_PCDS_PATCHABLE_IN_MODULE and Pcd.TokenCName in TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SET: + ModuleIsPatch = True + break + + if not ModuleIsPatch: + continue + # + # Module includes the patchable load fix address PCDs. + # It will be fixed up later. + # + PatchEfiImageList.append (OutputImageFile) + + # + # Get Top Memory address + # + ReservedRuntimeMemorySize = 0 + TopMemoryAddress = 0 + if self.LoadFixAddress == 0xFFFFFFFFFFFFFFFF: + TopMemoryAddress = 0 + else: + TopMemoryAddress = self.LoadFixAddress + if TopMemoryAddress < RtSize + BtSize + PeiSize: + EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS is too low to load driver") + + # + # Patch FixAddress related PCDs into EFI image + # + for EfiImage in PatchEfiImageList: + EfiImageMap = EfiImage.replace('.efi', '.map') + if not os.path.exists(EfiImageMap): + continue + # + # Get PCD offset in EFI image by GenPatchPcdTable function + # + PcdTable = parsePcdInfoFromMapFile(EfiImageMap, EfiImage) + # + # Patch real PCD value by PatchPcdValue tool + # + for PcdInfo in PcdTable: + ReturnValue = 0 + if PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_PEI_PAGE_SIZE: + ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_PEI_PAGE_SIZE_DATA_TYPE, str (PeiSize // 0x1000)) + elif PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_DXE_PAGE_SIZE: + ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_DXE_PAGE_SIZE_DATA_TYPE, str (BtSize // 0x1000)) + elif PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_RUNTIME_PAGE_SIZE: + ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_RUNTIME_PAGE_SIZE_DATA_TYPE, str (RtSize // 0x1000)) + elif PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SMM_PAGE_SIZE and len (SmmModuleList) > 0: + ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SMM_PAGE_SIZE_DATA_TYPE, str (SmmSize // 0x1000)) + if ReturnValue != 0: + EdkLogger.error("build", PARAMETER_INVALID, "Patch PCD value failed", ExtraData=ErrorInfo) + + MapBuffer.append('PEI_CODE_PAGE_NUMBER = 0x%x\n' % (PeiSize // 0x1000)) + MapBuffer.append('BOOT_CODE_PAGE_NUMBER = 0x%x\n' % (BtSize // 0x1000)) + MapBuffer.append('RUNTIME_CODE_PAGE_NUMBER = 0x%x\n' % (RtSize // 0x1000)) + if len (SmmModuleList) > 0: + MapBuffer.append('SMM_CODE_PAGE_NUMBER = 0x%x\n' % (SmmSize // 0x1000)) + + PeiBaseAddr = TopMemoryAddress - RtSize - BtSize + BtBaseAddr = TopMemoryAddress - RtSize + RtBaseAddr = TopMemoryAddress - ReservedRuntimeMemorySize + + self._RebaseModule (MapBuffer, PeiBaseAddr, PeiModuleList, TopMemoryAddress == 0) + self._RebaseModule (MapBuffer, BtBaseAddr, BtModuleList, TopMemoryAddress == 0) + self._RebaseModule (MapBuffer, RtBaseAddr, RtModuleList, TopMemoryAddress == 0) + self._RebaseModule (MapBuffer, 0x1000, SmmModuleList, AddrIsOffset=False, ModeIsSmm=True) + MapBuffer.append('\n\n') + sys.stdout.write ("\n") + sys.stdout.flush() + + ## Save platform Map file + # + def _SaveMapFile (self, MapBuffer, Wa): + # + # Map file path is got. + # + MapFilePath = os.path.join(Wa.BuildDir, Wa.Name + '.map') + # + # Save address map into MAP file. + # + SaveFileOnChange(MapFilePath, ''.join(MapBuffer), False) + if self.LoadFixAddress != 0: + sys.stdout.write ("\nLoad Module At Fix Address Map file can be found at %s\n" % (MapFilePath)) + sys.stdout.flush() + + ## Build active platform for different build targets and different tool chains + # + def _BuildPlatform(self): + SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False) + for BuildTarget in self.BuildTargetList: + GlobalData.gGlobalDefines['TARGET'] = BuildTarget + index = 0 + for ToolChain in self.ToolChainList: + GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain + GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain + GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index] + index += 1 + Wa = WorkspaceAutoGen( + self.WorkspaceDir, + self.PlatformFile, + BuildTarget, + ToolChain, + self.ArchList, + self.BuildDatabase, + self.TargetTxt, + self.ToolDef, + self.Fdf, + self.FdList, + self.FvList, + self.CapList, + self.SkuId, + self.UniFlag, + self.Progress + ) + self.Fdf = Wa.FdfFile + self.LoadFixAddress = Wa.Platform.LoadFixAddress + self.BuildReport.AddPlatformReport(Wa) + self.Progress.Stop("done!") + + # Add ffs build to makefile + CmdListDict = {} + if GlobalData.gEnableGenfdsMultiThread and self.Fdf: + CmdListDict = self._GenFfsCmd(Wa.ArchList) + + for Arch in Wa.ArchList: + PcdMaList = [] + GlobalData.gGlobalDefines['ARCH'] = Arch + Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch) + for Module in Pa.Platform.Modules: + # Get ModuleAutoGen object to generate C code file and makefile + Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe) + if Ma is None: + continue + if Ma.PcdIsDriver: + Ma.PlatformInfo = Pa + Ma.Workspace = Wa + PcdMaList.append(Ma) + self.BuildModules.append(Ma) + Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} + Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp} + self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict,PcdMaList=PcdMaList) + + # Create MAP file when Load Fix Address is enabled. + if self.Target in ["", "all", "fds"]: + for Arch in Wa.ArchList: + GlobalData.gGlobalDefines['ARCH'] = Arch + # + # Check whether the set fix address is above 4G for 32bit image. + # + if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000: + EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platform with IA32 or ARM arch modules") + # + # Get Module List + # + ModuleList = {} + for Pa in Wa.AutoGenObjectList: + for Ma in Pa.ModuleAutoGenList: + if Ma is None: + continue + if not Ma.IsLibrary: + ModuleList[Ma.Guid.upper()] = Ma + + MapBuffer = [] + if self.LoadFixAddress != 0: + # + # Rebase module to the preferred memory address before GenFds + # + self._CollectModuleMapBuffer(MapBuffer, ModuleList) + if self.Fdf: + # + # create FDS again for the updated EFI image + # + self._Build("fds", Wa) + # + # Create MAP file for all platform FVs after GenFds. + # + self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList) + # + # Save MAP buffer into MAP file. + # + self._SaveMapFile (MapBuffer, Wa) + self.CreateGuidedSectionToolsFile(Wa) + + ## Build active module for different build targets, different tool chains and different archs + # + def _BuildModule(self): + for BuildTarget in self.BuildTargetList: + GlobalData.gGlobalDefines['TARGET'] = BuildTarget + index = 0 + for ToolChain in self.ToolChainList: + WorkspaceAutoGenTime = time.time() + GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain + GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain + GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index] + index += 1 + # + # module build needs platform build information, so get platform + # AutoGen first + # + Wa = WorkspaceAutoGen( + self.WorkspaceDir, + self.PlatformFile, + BuildTarget, + ToolChain, + self.ArchList, + self.BuildDatabase, + self.TargetTxt, + self.ToolDef, + self.Fdf, + self.FdList, + self.FvList, + self.CapList, + self.SkuId, + self.UniFlag, + self.Progress, + self.ModuleFile + ) + self.Fdf = Wa.FdfFile + self.LoadFixAddress = Wa.Platform.LoadFixAddress + Wa.CreateMakeFile(False) + # Add ffs build to makefile + CmdListDict = None + if GlobalData.gEnableGenfdsMultiThread and self.Fdf: + CmdListDict = self._GenFfsCmd(Wa.ArchList) + + GlobalData.file_lock = mp.Lock() + GlobalData.FfsCmd = CmdListDict + + self.Progress.Stop("done!") + MaList = [] + ExitFlag = threading.Event() + ExitFlag.clear() + self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime))) + for Arch in Wa.ArchList: + AutoGenStart = time.time() + GlobalData.gGlobalDefines['ARCH'] = Arch + Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch) + for Module in Pa.Platform.Modules: + if self.ModuleFile.Dir == Module.Dir and self.ModuleFile.Name == Module.Name: + Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe) + if Ma is None: + continue + if Ma.PcdIsDriver: + Ma.PlatformInfo = Pa + Ma.Workspace = Wa + MaList.append(Ma) + + if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and self.Target in [None, "", "all"]: + if Ma.CanSkipbyPreMakeCache(): + continue + else: + self.PreMakeCacheMiss.add(Ma) + + # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds' + if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: + # for target which must generate AutoGen code and makefile + if not self.SkipAutoGen or self.Target == 'genc': + self.Progress.Start("Generating code") + Ma.CreateCodeFile(True) + self.Progress.Stop("done!") + if self.Target == "genc": + return True + if not self.SkipAutoGen or self.Target == 'genmake': + self.Progress.Start("Generating makefile") + if CmdListDict and self.Fdf and (Module.Path, Arch) in CmdListDict: + Ma.CreateMakeFile(True, CmdListDict[Module.Path, Arch]) + del CmdListDict[Module.Path, Arch] + else: + Ma.CreateMakeFile(True) + self.Progress.Stop("done!") + if self.Target == "genmake": + return True + + if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: + if Ma.CanSkipbyMakeCache(): + continue + else: + self.MakeCacheMiss.add(Ma) + + self.BuildModules.append(Ma) + self.AutoGenTime += int(round((time.time() - AutoGenStart))) + MakeStart = time.time() + for Ma in self.BuildModules: + if not Ma.IsBinaryModule: + Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target)) + # Break build if any build thread has error + if BuildTask.HasError(): + # we need a full version of makefile for platform + ExitFlag.set() + BuildTask.WaitForComplete() + Pa.CreateMakeFile(False) + EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule) + # Start task scheduler + if not BuildTask.IsOnGoing(): + BuildTask.StartScheduler(self.ThreadNumber, ExitFlag) + + # in case there's an interruption. we need a full version of makefile for platform + Pa.CreateMakeFile(False) + if BuildTask.HasError(): + EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule) + self.MakeTime += int(round((time.time() - MakeStart))) + + MakeContiue = time.time() + ExitFlag.set() + BuildTask.WaitForComplete() + self.CreateAsBuiltInf() + if GlobalData.gBinCacheDest: + self.GenDestCache() + elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource: + # Only for --hash + # Update PreMakeCacheChain files + self.GenLocalPreMakeCache() + self.BuildModules = [] + self.MakeTime += int(round((time.time() - MakeContiue))) + if BuildTask.HasError(): + EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule) + + self.BuildReport.AddPlatformReport(Wa, MaList) + if MaList == []: + EdkLogger.error( + 'build', + BUILD_ERROR, + "Module for [%s] is not a component of active platform."\ + " Please make sure that the ARCH and inf file path are"\ + " given in the same as in [%s]" % \ + (', '.join(Wa.ArchList), self.PlatformFile), + ExtraData=self.ModuleFile + ) + # Create MAP file when Load Fix Address is enabled. + if self.Target == "fds" and self.Fdf: + for Arch in Wa.ArchList: + # + # Check whether the set fix address is above 4G for 32bit image. + # + if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000: + EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules") + # + # Get Module List + # + ModuleList = {} + for Pa in Wa.AutoGenObjectList: + for Ma in Pa.ModuleAutoGenList: + if Ma is None: + continue + if not Ma.IsLibrary: + ModuleList[Ma.Guid.upper()] = Ma + + MapBuffer = [] + if self.LoadFixAddress != 0: + # + # Rebase module to the preferred memory address before GenFds + # + self._CollectModuleMapBuffer(MapBuffer, ModuleList) + # + # create FDS again for the updated EFI image + # + GenFdsStart = time.time() + self._Build("fds", Wa) + self.GenFdsTime += int(round((time.time() - GenFdsStart))) + # + # Create MAP file for all platform FVs after GenFds. + # + self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList) + # + # Save MAP buffer into MAP file. + # + self._SaveMapFile (MapBuffer, Wa) + + def _GenFfsCmd(self,ArchList): + # convert dictionary of Cmd:(Inf,Arch) + # to a new dictionary of (Inf,Arch):Cmd,Cmd,Cmd... + CmdSetDict = defaultdict(set) + GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, ArchList, GlobalData) + for Cmd in GenFfsDict: + tmpInf, tmpArch = GenFfsDict[Cmd] + CmdSetDict[tmpInf, tmpArch].add(Cmd) + return CmdSetDict + def VerifyAutoGenFiles(self): + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") + try: + with open(AutoGenIdFile) as fd: + lines = fd.readlines() + except: + return None + for line in lines: + if "Arch" in line: + ArchList = line.strip().split("=")[1].split("|") + if "BuildDir" in line: + BuildDir = line.split("=")[1].strip() + if "PlatformGuid" in line: + PlatformGuid = line.split("=")[1].strip() + GlobalVarList = [] + for arch in ArchList: + global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" % (str(PlatformGuid),arch)) + if not os.path.exists(global_var): + return None + GlobalVarList.append(global_var) + for global_var in GlobalVarList: + data_pipe = MemoryDataPipe() + data_pipe.load(global_var) + target = data_pipe.Get("P_Info").get("Target") + toolchain = data_pipe.Get("P_Info").get("ToolChain") + archlist = data_pipe.Get("P_Info").get("ArchList") + Arch = data_pipe.Get("P_Info").get("Arch") + active_p = data_pipe.Get("P_Info").get("ActivePlatform") + workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir") + PackagesPath = os.getenv("PACKAGES_PATH") + mws.setWs(workspacedir, PackagesPath) + LibraryBuildDirectoryList = data_pipe.Get("LibraryBuildDirectoryList") + ModuleBuildDirectoryList = data_pipe.Get("ModuleBuildDirectoryList") + + for m_build_dir in LibraryBuildDirectoryList: + if not os.path.exists(os.path.join(m_build_dir,self.MakeFileName)): + return None + for m_build_dir in ModuleBuildDirectoryList: + if not os.path.exists(os.path.join(m_build_dir,self.MakeFileName)): + return None + Wa = WorkSpaceInfo( + workspacedir,active_p,target,toolchain,archlist + ) + Pa = PlatformInfo(Wa, active_p, target, toolchain, Arch,data_pipe) + Wa.AutoGenObjectList.append(Pa) + return Wa + def SetupMakeSetting(self,Wa): + BuildModules = [] + for Pa in Wa.AutoGenObjectList: + for m in Pa._MbList: + ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget, Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe) + BuildModules.append(ma) + fdf_file = Wa.FlashDefinition + if fdf_file: + Fdf = FdfParser(fdf_file.Path) + Fdf.ParseFile() + GlobalData.gFdfParser = Fdf + if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict: + FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName] + for FdRegion in FdDict.RegionList: + if str(FdRegion.RegionType) == 'FILE' and self.Platform.VpdToolGuid in str(FdRegion.RegionDataList): + if int(FdRegion.Offset) % 8 != 0: + EdkLogger.error("build", FORMAT_INVALID, 'The VPD Base Address %s must be 8-byte aligned.' % (FdRegion.Offset)) + Wa.FdfProfile = Fdf.Profile + self.Fdf = Fdf + else: + self.Fdf = None + return BuildModules + + ## Build a platform in multi-thread mode + # + def PerformAutoGen(self,BuildTarget,ToolChain): + WorkspaceAutoGenTime = time.time() + Wa = WorkspaceAutoGen( + self.WorkspaceDir, + self.PlatformFile, + BuildTarget, + ToolChain, + self.ArchList, + self.BuildDatabase, + self.TargetTxt, + self.ToolDef, + self.Fdf, + self.FdList, + self.FvList, + self.CapList, + self.SkuId, + self.UniFlag, + self.Progress + ) + self.Fdf = Wa.FdfFile + self.LoadFixAddress = Wa.Platform.LoadFixAddress + self.BuildReport.AddPlatformReport(Wa) + Wa.CreateMakeFile(False) + + # Add ffs build to makefile + CmdListDict = {} + if GlobalData.gEnableGenfdsMultiThread and self.Fdf: + CmdListDict = self._GenFfsCmd(Wa.ArchList) + + self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime))) + BuildModules = [] + for Arch in Wa.ArchList: + PcdMaList = [] + AutoGenStart = time.time() + GlobalData.gGlobalDefines['ARCH'] = Arch + Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch) + if Pa is None: + continue + ModuleList = [] + for Inf in Pa.Platform.Modules: + ModuleList.append(Inf) + # Add the INF only list in FDF + if GlobalData.gFdfParser is not None: + for InfName in GlobalData.gFdfParser.Profile.InfList: + Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch) + if Inf in Pa.Platform.Modules: + continue + ModuleList.append(Inf) + Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} + Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp} + Pa.DataPipe.DataContainer = {"CommandTarget": self.Target} + Pa.CreateLibModuelDirs() + # Fetch the MakeFileName. + self.MakeFileName = Pa.MakeFileName + if not self.MakeFileName: + self.MakeFileName = Pa.MakeFile + + Pa.DataPipe.DataContainer = {"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList} + Pa.DataPipe.DataContainer = {"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList} + Pa.DataPipe.DataContainer = {"FdsCommandDict": Wa.GenFdsCommandDict} + # Prepare the cache share data for multiprocessing + Pa.DataPipe.DataContainer = {"gPlatformHashFile":GlobalData.gPlatformHashFile} + ModuleCodaFile = {} + for ma in Pa.ModuleAutoGenList: + ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFile.Path)] = [item.Target for item in ma.CodaTargetList] + Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile} + # ModuleList contains all driver modules only + for Module in ModuleList: + # Get ModuleAutoGen object to generate C code file and makefile + Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe) + if Ma is None: + continue + if Ma.PcdIsDriver: + Ma.PlatformInfo = Pa + Ma.Workspace = Wa + PcdMaList.append(Ma) + self.AllDrivers.add(Ma) + self.AllModules.add(Ma) + + mqueue = mp.Queue() + cqueue = mp.Queue() + for m in Pa.GetAllModuleInfo: + mqueue.put(m) + module_file,module_root,module_path,module_basename,\ + module_originalpath,module_arch,IsLib = m + Ma = ModuleAutoGen(Wa, PathClass(module_path, Wa), BuildTarget,\ + ToolChain, Arch, self.PlatformFile,Pa.DataPipe) + self.AllModules.add(Ma) + data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) + Pa.DataPipe.dump(data_pipe_file) + + mqueue.put((None,None,None,None,None,None,None)) + autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList, cqueue) + + if not autogen_rt: + self.AutoGenMgr.TerminateWorkers() + self.AutoGenMgr.join(1) + raise FatalError(errorcode) + + if GlobalData.gUseHashCache: + for item in GlobalData.gModuleAllCacheStatus: + (MetaFilePath, Arch, CacheStr, Status) = item + Ma = ModuleAutoGen(Wa, PathClass(MetaFilePath, Wa), BuildTarget,\ + ToolChain, Arch, self.PlatformFile,Pa.DataPipe) + if CacheStr == "PreMakeCache" and Status == False: + self.PreMakeCacheMiss.add(Ma) + if CacheStr == "PreMakeCache" and Status == True: + self.PreMakeCacheHit.add(Ma) + GlobalData.gModuleCacheHit.add(Ma) + if CacheStr == "MakeCache" and Status == False: + self.MakeCacheMiss.add(Ma) + if CacheStr == "MakeCache" and Status == True: + self.MakeCacheHit.add(Ma) + GlobalData.gModuleCacheHit.add(Ma) + self.AutoGenTime += int(round((time.time() - AutoGenStart))) + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") + with open(AutoGenIdFile,"w") as fw: + fw.write("Arch=%s\n" % "|".join((Wa.ArchList))) + fw.write("BuildDir=%s\n" % Wa.BuildDir) + fw.write("PlatformGuid=%s\n" % str(Wa.AutoGenObjectList[0].Guid)) + + if GlobalData.gBinCacheSource: + BuildModules.extend(self.MakeCacheMiss) + elif GlobalData.gUseHashCache and not GlobalData.gBinCacheDest: + BuildModules.extend(self.PreMakeCacheMiss) + else: + BuildModules.extend(self.AllDrivers) + + self.Progress.Stop("done!") + return Wa, BuildModules + + def _MultiThreadBuildPlatform(self): + SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False) + for BuildTarget in self.BuildTargetList: + GlobalData.gGlobalDefines['TARGET'] = BuildTarget + index = 0 + for ToolChain in self.ToolChainList: + resetFdsGlobalVariable() + GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain + GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain + GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index] + index += 1 + ExitFlag = threading.Event() + ExitFlag.clear() + if self.SkipAutoGen: + Wa = self.VerifyAutoGenFiles() + if Wa is None: + self.SkipAutoGen = False + Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain) + else: + GlobalData.gAutoGenPhase = True + self.BuildModules = self.SetupMakeSetting(Wa) + else: + Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain) + Pa = Wa.AutoGenObjectList[0] + GlobalData.gAutoGenPhase = False + + if GlobalData.gBinCacheSource: + EdkLogger.quiet("[cache Summary]: Total module num: %s" % len(self.AllModules)) + EdkLogger.quiet("[cache Summary]: PreMakecache miss num: %s " % len(self.PreMakeCacheMiss)) + EdkLogger.quiet("[cache Summary]: Makecache miss num: %s " % len(self.MakeCacheMiss)) + + for Arch in Wa.ArchList: + MakeStart = time.time() + for Ma in set(self.BuildModules): + # Generate build task for the module + if not Ma.IsBinaryModule: + Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target)) + # Break build if any build thread has error + if BuildTask.HasError(): + # we need a full version of makefile for platform + ExitFlag.set() + BuildTask.WaitForComplete() + Pa.CreateMakeFile(False) + EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule) + # Start task scheduler + if not BuildTask.IsOnGoing(): + BuildTask.StartScheduler(self.ThreadNumber, ExitFlag) + + # in case there's an interruption. we need a full version of makefile for platform + + if BuildTask.HasError(): + EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule) + self.MakeTime += int(round((time.time() - MakeStart))) + + MakeContiue = time.time() + # + # + # All modules have been put in build tasks queue. Tell task scheduler + # to exit if all tasks are completed + # + ExitFlag.set() + BuildTask.WaitForComplete() + if GlobalData.gBinCacheDest: + self.GenDestCache() + elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource: + # Only for --hash + # Update PreMakeCacheChain files + self.GenLocalPreMakeCache() + # + # Get Module List + # + ModuleList = {ma.Guid.upper(): ma for ma in self.BuildModules} + self.BuildModules = [] + self.MakeTime += int(round((time.time() - MakeContiue))) + # + # Check for build error, and raise exception if one + # has been signaled. + # + if BuildTask.HasError(): + EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule) + + # Create MAP file when Load Fix Address is enabled. + if self.Target in ["", "all", "fds"]: + for Arch in Wa.ArchList: + # + # Check whether the set fix address is above 4G for 32bit image. + # + if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000: + EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules") + + # + # Rebase module to the preferred memory address before GenFds + # + MapBuffer = [] + if self.LoadFixAddress != 0: + self._CollectModuleMapBuffer(MapBuffer, ModuleList) + + if self.Fdf: + # + # Generate FD image if there's a FDF file found + # + GenFdsStart = time.time() + if GenFdsApi(Wa.GenFdsCommandDict, self.Db): + EdkLogger.error("build", COMMAND_FAILURE) + Threshold = self.GetFreeSizeThreshold() + if Threshold: + self.CheckFreeSizeThreshold(Threshold, Wa.FvDir) + + # + # Create MAP file for all platform FVs after GenFds. + # + self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList) + self.GenFdsTime += int(round((time.time() - GenFdsStart))) + # + # Save MAP buffer into MAP file. + # + self._SaveMapFile(MapBuffer, Wa) + self.CreateGuidedSectionToolsFile(Wa) + + ## GetFreeSizeThreshold() + # + # @retval int Threshold value + # + def GetFreeSizeThreshold(self): + Threshold = None + Threshold_Str = GlobalData.gCommandLineDefines.get('FV_SPARE_SPACE_THRESHOLD') + if Threshold_Str: + try: + if Threshold_Str.lower().startswith('0x'): + Threshold = int(Threshold_Str, 16) + else: + Threshold = int(Threshold_Str) + except: + EdkLogger.warn("build", 'incorrect value for FV_SPARE_SPACE_THRESHOLD %s.Only decimal or hex format is allowed.' % Threshold_Str) + return Threshold + + def CheckFreeSizeThreshold(self, Threshold=None, FvDir=None): + if not isinstance(Threshold, int): + return + if not isinstance(FvDir, str) or not FvDir: + return + FdfParserObject = GlobalData.gFdfParser + FvRegionNameList = [FvName for FvName in FdfParserObject.Profile.FvDict if FdfParserObject.Profile.FvDict[FvName].FvRegionInFD] + for FvName in FdfParserObject.Profile.FvDict: + if FvName in FvRegionNameList: + FvSpaceInfoFileName = os.path.join(FvDir, FvName.upper() + '.Fv.map') + if os.path.exists(FvSpaceInfoFileName): + FileLinesList = getlines(FvSpaceInfoFileName) + for Line in FileLinesList: + NameValue = Line.split('=') + if len(NameValue) == 2 and NameValue[0].strip() == 'EFI_FV_SPACE_SIZE': + FreeSizeValue = int(NameValue[1].strip(), 0) + if FreeSizeValue < Threshold: + EdkLogger.error("build", FV_FREESIZE_ERROR, + '%s FV free space %d is not enough to meet with the required spare space %d set by -D FV_SPARE_SPACE_THRESHOLD option.' % ( + FvName, FreeSizeValue, Threshold)) + break + + ## Generate GuidedSectionTools.txt in the FV directories. + # + def CreateGuidedSectionToolsFile(self,Wa): + for BuildTarget in self.BuildTargetList: + for ToolChain in self.ToolChainList: + FvDir = Wa.FvDir + if not os.path.exists(FvDir): + continue + for Arch in self.ArchList: + guidList = [] + tooldefguidList = [] + guidAttribs = [] + for Platform in Wa.AutoGenObjectList: + if Platform.BuildTarget != BuildTarget: + continue + if Platform.ToolChain != ToolChain: + continue + if Platform.Arch != Arch: + continue + if hasattr (Platform, 'BuildOption'): + for Tool in Platform.BuildOption: + if 'GUID' in Platform.BuildOption[Tool]: + if 'PATH' in Platform.BuildOption[Tool]: + value = Platform.BuildOption[Tool]['GUID'] + if value in guidList: + EdkLogger.error("build", FORMAT_INVALID, "Duplicate GUID value %s used with Tool %s in DSC [BuildOptions]." % (value, Tool)) + path = Platform.BuildOption[Tool]['PATH'] + guidList.append(value) + guidAttribs.append((value, Tool, path)) + for Tool in Platform.ToolDefinition: + if 'GUID' in Platform.ToolDefinition[Tool]: + if 'PATH' in Platform.ToolDefinition[Tool]: + value = Platform.ToolDefinition[Tool]['GUID'] + if value in tooldefguidList: + EdkLogger.error("build", FORMAT_INVALID, "Duplicate GUID value %s used with Tool %s in tools_def.txt." % (value, Tool)) + tooldefguidList.append(value) + if value in guidList: + # Already added by platform + continue + path = Platform.ToolDefinition[Tool]['PATH'] + guidList.append(value) + guidAttribs.append((value, Tool, path)) + # Sort by GuidTool name + guidAttribs = sorted (guidAttribs, key=lambda x: x[1]) + # Write out GuidedSecTools.txt + toolsFile = os.path.join(FvDir, 'GuidedSectionTools.txt') + toolsFile = open(toolsFile, 'wt') + for guidedSectionTool in guidAttribs: + print(' '.join(guidedSectionTool), file=toolsFile) + toolsFile.close() + + ## Returns the real path of the tool. + # + def GetRealPathOfTool (self, tool): + if os.path.exists(tool): + return os.path.realpath(tool) + return tool + + ## Launch the module or platform build + # + def Launch(self): + self.AllDrivers = set() + self.AllModules = set() + self.PreMakeCacheMiss = set() + self.PreMakeCacheHit = set() + self.MakeCacheMiss = set() + self.MakeCacheHit = set() + if not self.ModuleFile: + if not self.SpawnMode or self.Target not in ["", "all"]: + self.SpawnMode = False + self._BuildPlatform() + else: + self._MultiThreadBuildPlatform() + else: + self.SpawnMode = False + self._BuildModule() + + if self.Target == 'cleanall': + RemoveDirectory(os.path.dirname(GlobalData.gDatabasePath), True) + + def CreateAsBuiltInf(self): + for Module in self.BuildModules: + Module.CreateAsBuiltInf() + + def GenDestCache(self): + for Module in self.AllModules: + Module.GenPreMakefileHashList() + Module.GenMakefileHashList() + Module.CopyModuleToCache() + + def GenLocalPreMakeCache(self): + for Module in self.PreMakeCacheMiss: + Module.GenPreMakefileHashList() + + ## Do some clean-up works when error occurred + def Relinquish(self): + OldLogLevel = EdkLogger.GetLevel() + EdkLogger.SetLevel(EdkLogger.ERROR) + Utils.Progressor.Abort() + if self.SpawnMode == True: + BuildTask.Abort() + EdkLogger.SetLevel(OldLogLevel) + +def ParseDefines(DefineList=[]): + DefineDict = {} + if DefineList is not None: + for Define in DefineList: + DefineTokenList = Define.split("=", 1) + if not GlobalData.gMacroNamePattern.match(DefineTokenList[0]): + EdkLogger.error('build', FORMAT_INVALID, + "The macro name must be in the pattern [A-Z][A-Z0-9_]*", + ExtraData=DefineTokenList[0]) + + if len(DefineTokenList) == 1: + DefineDict[DefineTokenList[0]] = "TRUE" + else: + DefineDict[DefineTokenList[0]] = DefineTokenList[1].strip() + return DefineDict + + + +def LogBuildTime(Time): + if Time: + TimeDurStr = '' + TimeDur = time.gmtime(Time) + if TimeDur.tm_yday > 1: + TimeDurStr = time.strftime("%H:%M:%S", TimeDur) + ", %d day(s)" % (TimeDur.tm_yday - 1) + else: + TimeDurStr = time.strftime("%H:%M:%S", TimeDur) + return TimeDurStr + else: + return None +def ThreadNum(): + OptionParser = MyOptionParser() + if not OptionParser.BuildOption and not OptionParser.BuildTarget: + OptionParser.GetOption() + BuildOption, BuildTarget = OptionParser.BuildOption, OptionParser.BuildTarget + ThreadNumber = BuildOption.ThreadNumber + GlobalData.gCmdConfDir = BuildOption.ConfDirectory + if ThreadNumber is None: + TargetObj = TargetTxtDict() + ThreadNumber = TargetObj.Target.TargetTxtDictionary[TAB_TAT_DEFINES_MAX_CONCURRENT_THREAD_NUMBER] + if ThreadNumber == '': + ThreadNumber = 0 + else: + ThreadNumber = int(ThreadNumber, 0) + + if ThreadNumber == 0: + try: + ThreadNumber = multiprocessing.cpu_count() + except (ImportError, NotImplementedError): + ThreadNumber = 1 + return ThreadNumber +## Tool entrance method +# +# This method mainly dispatch specific methods per the command line options. +# If no error found, return zero value so the caller of this tool can know +# if it's executed successfully or not. +# +# @retval 0 Tool was successful +# @retval 1 Tool failed +# +LogQMaxSize = ThreadNum() * 10 +def Main(): + StartTime = time.time() + + # + # Create a log Queue + # + LogQ = mp.Queue(LogQMaxSize) + # Initialize log system + EdkLogger.LogClientInitialize(LogQ) + GlobalData.gCommand = sys.argv[1:] + # + # Parse the options and args + # + OptionParser = MyOptionParser() + if not OptionParser.BuildOption and not OptionParser.BuildTarget: + OptionParser.GetOption() + Option, Target = OptionParser.BuildOption, OptionParser.BuildTarget + GlobalData.gOptions = Option + GlobalData.gCaseInsensitive = Option.CaseInsensitive + + # Set log level + LogLevel = EdkLogger.INFO + if Option.verbose is not None: + EdkLogger.SetLevel(EdkLogger.VERBOSE) + LogLevel = EdkLogger.VERBOSE + elif Option.quiet is not None: + EdkLogger.SetLevel(EdkLogger.QUIET) + LogLevel = EdkLogger.QUIET + elif Option.debug is not None: + EdkLogger.SetLevel(Option.debug + 1) + LogLevel = Option.debug + 1 + else: + EdkLogger.SetLevel(EdkLogger.INFO) + + if Option.WarningAsError == True: + EdkLogger.SetWarningAsError() + Log_Agent = LogAgent(LogQ,LogLevel,Option.LogFile) + Log_Agent.start() + + if platform.platform().find("Windows") >= 0: + GlobalData.gIsWindows = True + else: + GlobalData.gIsWindows = False + + EdkLogger.quiet("Build environment: %s" % platform.platform()) + EdkLogger.quiet(time.strftime("Build start time: %H:%M:%S, %b.%d %Y\n", time.localtime())); + ReturnCode = 0 + MyBuild = None + BuildError = True + try: + if len(Target) == 0: + Target = "all" + elif len(Target) >= 2: + EdkLogger.error("build", OPTION_NOT_SUPPORTED, "More than one targets are not supported.", + ExtraData="Please select one of: %s" % (' '.join(gSupportedTarget))) + else: + Target = Target[0].lower() + + if Target not in gSupportedTarget: + EdkLogger.error("build", OPTION_NOT_SUPPORTED, "Not supported target [%s]." % Target, + ExtraData="Please select one of: %s" % (' '.join(gSupportedTarget))) + + # + # Check environment variable: EDK_TOOLS_PATH, WORKSPACE, PATH + # + CheckEnvVariable() + GlobalData.gCommandLineDefines.update(ParseDefines(Option.Macros)) + + Workspace = os.getenv("WORKSPACE") + # + # Get files real name in workspace dir + # + GlobalData.gAllFiles = Utils.DirCache(Workspace) + + WorkingDirectory = os.getcwd() + if not Option.ModuleFile: + FileList = glob.glob(os.path.normpath(os.path.join(WorkingDirectory, '*.inf'))) + FileNum = len(FileList) + if FileNum >= 2: + EdkLogger.error("build", OPTION_NOT_SUPPORTED, "There are %d INF files in %s." % (FileNum, WorkingDirectory), + ExtraData="Please use '-m <INF_FILE_PATH>' switch to choose one.") + elif FileNum == 1: + Option.ModuleFile = NormFile(FileList[0], Workspace) + + if Option.ModuleFile: + if os.path.isabs (Option.ModuleFile): + if os.path.normcase (os.path.normpath(Option.ModuleFile)).find (Workspace) == 0: + Option.ModuleFile = NormFile(os.path.normpath(Option.ModuleFile), Workspace) + Option.ModuleFile = PathClass(Option.ModuleFile, Workspace) + ErrorCode, ErrorInfo = Option.ModuleFile.Validate(".inf", False) + if ErrorCode != 0: + EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo) + + if Option.PlatformFile is not None: + if os.path.isabs (Option.PlatformFile): + if os.path.normcase (os.path.normpath(Option.PlatformFile)).find (Workspace) == 0: + Option.PlatformFile = NormFile(os.path.normpath(Option.PlatformFile), Workspace) + Option.PlatformFile = PathClass(Option.PlatformFile, Workspace) + + if Option.FdfFile is not None: + if os.path.isabs (Option.FdfFile): + if os.path.normcase (os.path.normpath(Option.FdfFile)).find (Workspace) == 0: + Option.FdfFile = NormFile(os.path.normpath(Option.FdfFile), Workspace) + Option.FdfFile = PathClass(Option.FdfFile, Workspace) + ErrorCode, ErrorInfo = Option.FdfFile.Validate(".fdf", False) + if ErrorCode != 0: + EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo) + + if Option.Flag is not None and Option.Flag not in ['-c', '-s']: + EdkLogger.error("build", OPTION_VALUE_INVALID, "UNI flag must be one of -c or -s") + + MyBuild = Build(Target, Workspace, Option,LogQ) + GlobalData.gCommandLineDefines['ARCH'] = ' '.join(MyBuild.ArchList) + if not (MyBuild.LaunchPrebuildFlag and os.path.exists(MyBuild.PlatformBuildPath)): + MyBuild.Launch() + + # + # All job done, no error found and no exception raised + # + BuildError = False + except FatalError as X: + if MyBuild is not None: + # for multi-thread build exits safely + MyBuild.Relinquish() + if Option is not None and Option.debug is not None: + EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc()) + ReturnCode = X.args[0] + except Warning as X: + # error from Fdf parser + if MyBuild is not None: + # for multi-thread build exits safely + MyBuild.Relinquish() + if Option is not None and Option.debug is not None: + EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc()) + else: + EdkLogger.error(X.ToolName, FORMAT_INVALID, File=X.FileName, Line=X.LineNumber, ExtraData=X.Message, RaiseError=False) + ReturnCode = FORMAT_INVALID + except KeyboardInterrupt: + if MyBuild is not None: + + # for multi-thread build exits safely + MyBuild.Relinquish() + ReturnCode = ABORT_ERROR + if Option is not None and Option.debug is not None: + EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc()) + except: + if MyBuild is not None: + # for multi-thread build exits safely + MyBuild.Relinquish() + + # try to get the meta-file from the object causing exception + Tb = sys.exc_info()[-1] + MetaFile = GlobalData.gProcessingFile + while Tb is not None: + if 'self' in Tb.tb_frame.f_locals and hasattr(Tb.tb_frame.f_locals['self'], 'MetaFile'): + MetaFile = Tb.tb_frame.f_locals['self'].MetaFile + Tb = Tb.tb_next + EdkLogger.error( + "\nbuild", + CODE_ERROR, + "Unknown fatal error when processing [%s]" % MetaFile, + ExtraData="\n(Please send email to %s for help, attaching following call stack trace!)\n" % MSG_EDKII_MAIL_ADDR, + RaiseError=False + ) + EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc()) + ReturnCode = CODE_ERROR + finally: + Utils.Progressor.Abort() + Utils.ClearDuplicatedInf() + + if ReturnCode == 0: + try: + MyBuild.LaunchPostbuild() + Conclusion = "Done" + except: + Conclusion = "Failed" + ReturnCode = POSTBUILD_ERROR + elif ReturnCode == ABORT_ERROR: + Conclusion = "Aborted" + else: + Conclusion = "Failed" + FinishTime = time.time() + BuildDuration = time.gmtime(int(round(FinishTime - StartTime))) + BuildDurationStr = "" + if BuildDuration.tm_yday > 1: + BuildDurationStr = time.strftime("%H:%M:%S", BuildDuration) + ", %d day(s)" % (BuildDuration.tm_yday - 1) + else: + BuildDurationStr = time.strftime("%H:%M:%S", BuildDuration) + if MyBuild is not None: + if not BuildError: + MyBuild.BuildReport.GenerateReport(BuildDurationStr, LogBuildTime(MyBuild.AutoGenTime), LogBuildTime(MyBuild.MakeTime), LogBuildTime(MyBuild.GenFdsTime)) + + EdkLogger.SetLevel(EdkLogger.QUIET) + EdkLogger.quiet("\n- %s -" % Conclusion) + EdkLogger.quiet(time.strftime("Build end time: %H:%M:%S, %b.%d %Y", time.localtime())) + EdkLogger.quiet("Build total time: %s\n" % BuildDurationStr) + Log_Agent.kill() + Log_Agent.join() + return ReturnCode + +if __name__ == '__main__': + try: + mp.set_start_method('spawn') + except: + pass + r = Main() + ## 0-127 is a safe return range, and 1 is a standard default error + if r < 0 or r > 127: r = 1 + sys.exit(r) |