diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 16:49:04 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 16:49:04 +0000 |
commit | 16f504a9dca3fe3b70568f67b7d41241ae485288 (patch) | |
tree | c60f36ada0496ba928b7161059ba5ab1ab224f9d /src/VBox/Devices/EFI/Firmware/.pytool | |
parent | Initial commit. (diff) | |
download | virtualbox-16f504a9dca3fe3b70568f67b7d41241ae485288.tar.xz virtualbox-16f504a9dca3fe3b70568f67b7d41241ae485288.zip |
Adding upstream version 7.0.6-dfsg.upstream/7.0.6-dfsgupstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/VBox/Devices/EFI/Firmware/.pytool')
36 files changed, 3054 insertions, 0 deletions
diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/CISettings.py b/src/VBox/Devices/EFI/Firmware/.pytool/CISettings.py new file mode 100755 index 00000000..ed0b4732 --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/CISettings.py @@ -0,0 +1,227 @@ +# @file +# +# Copyright (c) Microsoft Corporation. +# Copyright (c) 2020, Hewlett Packard Enterprise Development LP. All rights reserved.<BR> +# Copyright (c) 2020 - 2021, ARM Limited. All rights reserved.<BR> +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +import os +import logging +from edk2toolext.environment import shell_environment +from edk2toolext.invocables.edk2_ci_build import CiBuildSettingsManager +from edk2toolext.invocables.edk2_setup import SetupSettingsManager, RequiredSubmodule +from edk2toolext.invocables.edk2_update import UpdateSettingsManager +from edk2toolext.invocables.edk2_pr_eval import PrEvalSettingsManager +from edk2toollib.utility_functions import GetHostInfo + + +class Settings(CiBuildSettingsManager, UpdateSettingsManager, SetupSettingsManager, PrEvalSettingsManager): + + def __init__(self): + self.ActualPackages = [] + self.ActualTargets = [] + self.ActualArchitectures = [] + self.ActualToolChainTag = "" + self.UseBuiltInBaseTools = None + self.ActualScopes = None + + # ####################################################################################### # + # Extra CmdLine configuration # + # ####################################################################################### # + + def AddCommandLineOptions(self, parserObj): + group = parserObj.add_mutually_exclusive_group() + group.add_argument("-force_piptools", "--fpt", dest="force_piptools", action="store_true", default=False, help="Force the system to use pip tools") + group.add_argument("-no_piptools", "--npt", dest="no_piptools", action="store_true", default=False, help="Force the system to not use pip tools") + + def RetrieveCommandLineOptions(self, args): + super().RetrieveCommandLineOptions(args) + if args.force_piptools: + self.UseBuiltInBaseTools = True + if args.no_piptools: + self.UseBuiltInBaseTools = False + + # ####################################################################################### # + # Default Support for this Ci Build # + # ####################################################################################### # + + def GetPackagesSupported(self): + ''' return iterable of edk2 packages supported by this build. + These should be edk2 workspace relative paths ''' + + return ("ArmPkg", + "ArmPlatformPkg", + "ArmVirtPkg", + "DynamicTablesPkg", + "EmulatorPkg", + "MdePkg", + "MdeModulePkg", + "NetworkPkg", + "PcAtChipsetPkg", + "SecurityPkg", + "UefiCpuPkg", + "FmpDevicePkg", + "ShellPkg", + "StandaloneMmPkg", + "FatPkg", + "CryptoPkg", + "UnitTestFrameworkPkg", + "OvmfPkg", + "RedfishPkg" + ) + + def GetArchitecturesSupported(self): + ''' return iterable of edk2 architectures supported by this build ''' + return ( + "IA32", + "X64", + "ARM", + "AARCH64", + "RISCV64") + + def GetTargetsSupported(self): + ''' return iterable of edk2 target tags supported by this build ''' + return ("DEBUG", "RELEASE", "NO-TARGET", "NOOPT") + + # ####################################################################################### # + # Verify and Save requested Ci Build Config # + # ####################################################################################### # + + def SetPackages(self, list_of_requested_packages): + ''' Confirm the requested package list is valid and configure SettingsManager + to build the requested packages. + + Raise UnsupportedException if a requested_package is not supported + ''' + unsupported = set(list_of_requested_packages) - \ + set(self.GetPackagesSupported()) + if(len(unsupported) > 0): + logging.critical( + "Unsupported Package Requested: " + " ".join(unsupported)) + raise Exception("Unsupported Package Requested: " + + " ".join(unsupported)) + self.ActualPackages = list_of_requested_packages + + def SetArchitectures(self, list_of_requested_architectures): + ''' Confirm the requests architecture list is valid and configure SettingsManager + to run only the requested architectures. + + Raise Exception if a list_of_requested_architectures is not supported + ''' + unsupported = set(list_of_requested_architectures) - \ + set(self.GetArchitecturesSupported()) + if(len(unsupported) > 0): + logging.critical( + "Unsupported Architecture Requested: " + " ".join(unsupported)) + raise Exception( + "Unsupported Architecture Requested: " + " ".join(unsupported)) + self.ActualArchitectures = list_of_requested_architectures + + def SetTargets(self, list_of_requested_target): + ''' Confirm the request target list is valid and configure SettingsManager + to run only the requested targets. + + Raise UnsupportedException if a requested_target is not supported + ''' + unsupported = set(list_of_requested_target) - \ + set(self.GetTargetsSupported()) + if(len(unsupported) > 0): + logging.critical( + "Unsupported Targets Requested: " + " ".join(unsupported)) + raise Exception("Unsupported Targets Requested: " + + " ".join(unsupported)) + self.ActualTargets = list_of_requested_target + + # ####################################################################################### # + # Actual Configuration for Ci Build # + # ####################################################################################### # + + def GetActiveScopes(self): + ''' return tuple containing scopes that should be active for this process ''' + if self.ActualScopes is None: + scopes = ("cibuild", "edk2-build", "host-based-test") + + self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "") + + is_linux = GetHostInfo().os.upper() == "LINUX" + + if self.UseBuiltInBaseTools is None: + is_linux = GetHostInfo().os.upper() == "LINUX" + # try and import the pip module for basetools + try: + import edk2basetools + self.UseBuiltInBaseTools = True + except ImportError: + self.UseBuiltInBaseTools = False + pass + + if self.UseBuiltInBaseTools == True: + scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',) + logging.warning("Using Pip Tools based BaseTools") + else: + logging.warning("Falling back to using in-tree BaseTools") + + if is_linux and self.ActualToolChainTag.upper().startswith("GCC"): + if "AARCH64" in self.ActualArchitectures: + scopes += ("gcc_aarch64_linux",) + if "ARM" in self.ActualArchitectures: + scopes += ("gcc_arm_linux",) + if "RISCV64" in self.ActualArchitectures: + scopes += ("gcc_riscv64_unknown",) + self.ActualScopes = scopes + return self.ActualScopes + + def GetRequiredSubmodules(self): + ''' return iterable containing RequiredSubmodule objects. + If no RequiredSubmodules return an empty iterable + ''' + rs = [] + rs.append(RequiredSubmodule( + "ArmPkg/Library/ArmSoftFloatLib/berkeley-softfloat-3", False)) + rs.append(RequiredSubmodule( + "CryptoPkg/Library/OpensslLib/openssl", False)) + rs.append(RequiredSubmodule( + "UnitTestFrameworkPkg/Library/CmockaLib/cmocka", False)) + rs.append(RequiredSubmodule( + "MdeModulePkg/Universal/RegularExpressionDxe/oniguruma", False)) + rs.append(RequiredSubmodule( + "MdeModulePkg/Library/BrotliCustomDecompressLib/brotli", False)) + rs.append(RequiredSubmodule( + "BaseTools/Source/C/BrotliCompress/brotli", False)) + rs.append(RequiredSubmodule( + "RedfishPkg/Library/JsonLib/jansson", False)) + return rs + + def GetName(self): + return "Edk2" + + def GetDependencies(self): + return [ + ] + + def GetPackagesPath(self): + return () + + def GetWorkspaceRoot(self): + ''' get WorkspacePath ''' + return os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + def FilterPackagesToTest(self, changedFilesList: list, potentialPackagesList: list) -> list: + ''' Filter potential packages to test based on changed files. ''' + build_these_packages = [] + possible_packages = potentialPackagesList.copy() + for f in changedFilesList: + # split each part of path for comparison later + nodes = f.split("/") + + # python file change in .pytool folder causes building all + if f.endswith(".py") and ".pytool" in nodes: + build_these_packages = possible_packages + break + + # BaseTools files that might change the build + if "BaseTools" in nodes: + if os.path.splitext(f) not in [".txt", ".md"]: + build_these_packages = possible_packages + break + return build_these_packages diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/CharEncodingCheck/CharEncodingCheck.py b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/CharEncodingCheck/CharEncodingCheck.py new file mode 100755 index 00000000..1b0e2397 --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/CharEncodingCheck/CharEncodingCheck.py @@ -0,0 +1,118 @@ +# @file CharEncodingCheck.py +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + + +import os +import logging +from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin +from edk2toolext.environment.var_dict import VarDict + +## +# map +## +EcodingMap = { + ".md": 'utf-8', + ".dsc": 'utf-8', + ".dec": 'utf-8', + ".c": 'utf-8', + ".h": 'utf-8', + ".asm": 'utf-8', + ".masm": 'utf-8', + ".nasm": 'utf-8', + ".s": 'utf-8', + ".inf": 'utf-8', + ".asl": 'utf-8', + ".uni": 'utf-8', + ".py": 'utf-8' +} + + +class CharEncodingCheck(ICiBuildPlugin): + """ + A CiBuildPlugin that scans each file in the code tree and confirms the encoding is correct. + + Configuration options: + "CharEncodingCheck": { + "IgnoreFiles": [] + } + """ + + def GetTestName(self, packagename: str, environment: VarDict) -> tuple: + """ Provide the testcase name and classname for use in reporting + testclassname: a descriptive string for the testcase can include whitespace + classname: should be patterned <packagename>.<plugin>.<optionally any unique condition> + + Args: + packagename: string containing name of package to build + environment: The VarDict for the test to run in + Returns: + a tuple containing the testcase name and the classname + (testcasename, classname) + """ + return ("Check for valid file encoding for " + packagename, packagename + ".CharEncodingCheck") + + ## + # External function of plugin. This function is used to perform the task of the ci_build_plugin Plugin + # + # - package is the edk2 path to package. This means workspace/packagepath relative. + # - edk2path object configured with workspace and packages path + # - PkgConfig Object (dict) for the pkg + # - EnvConfig Object + # - Plugin Manager Instance + # - Plugin Helper Obj Instance + # - Junit Logger + # - output_stream the StringIO output stream from this plugin via logging + def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None): + overall_status = 0 + files_tested = 0 + + abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagename) + + if abs_pkg_path is None: + tc.SetSkipped() + tc.LogStdError("No Package folder {0}".format(abs_pkg_path)) + return 0 + + for (ext, enc) in EcodingMap.items(): + files = self.WalkDirectoryForExtension([ext], abs_pkg_path) + files = [Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(x) for x in files] # make edk2relative path so can process ignores + + if "IgnoreFiles" in pkgconfig: + for a in pkgconfig["IgnoreFiles"]: + a = a.replace(os.sep, "/") + try: + tc.LogStdOut("Ignoring File {0}".format(a)) + files.remove(a) + except: + tc.LogStdError("CharEncodingCheck.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a)) + logging.info("CharEncodingCheck.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a)) + + files = [Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(x) for x in files] + for a in files: + files_tested += 1 + if(self.TestEncodingOk(a, enc)): + logging.debug("File {0} Passed Encoding Check {1}".format(a, enc)) + else: + tc.LogStdError("Encoding Failure in {0}. Not {1}".format(a, enc)) + overall_status += 1 + + tc.LogStdOut("Tested Encoding on {0} files".format(files_tested)) + if overall_status != 0: + tc.SetFailed("CharEncoding {0} Failed. Errors {1}".format(packagename, overall_status), "CHAR_ENCODING_CHECK_FAILED") + else: + tc.SetSuccess() + return overall_status + + def TestEncodingOk(self, apath, encodingValue): + try: + with open(apath, "rb") as fobj: + fobj.read().decode(encodingValue) + except Exception as exp: + logging.error("Encoding failure: file: {0} type: {1}".format(apath, encodingValue)) + logging.debug("EXCEPTION: while processing {1} - {0}".format(exp, apath)) + return False + + return True diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/CharEncodingCheck/CharEncodingCheck_plug_in.yaml b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/CharEncodingCheck/CharEncodingCheck_plug_in.yaml new file mode 100644 index 00000000..f5f126d9 --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/CharEncodingCheck/CharEncodingCheck_plug_in.yaml @@ -0,0 +1,11 @@ +## @file +# CiBuildPlugin used to check char encoding +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "cibuild", + "name": "Char Encoding Check Test", + "module": "CharEncodingCheck" +} diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/CharEncodingCheck/Readme.md b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/CharEncodingCheck/Readme.md new file mode 100644 index 00000000..0e6cbbea --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/CharEncodingCheck/Readme.md @@ -0,0 +1,18 @@ +# Character Encoding Check Plugin + +This CiBuildPlugin scans all the files in a package to make sure each file is +correctly encoded and all characters can be read. Improper encoding causes +tools to fail in some situations especially in different locals. + +## Configuration + +The plugin can be configured to ignore certain files. + +``` yaml +"CharEncodingCheck": { + "IgnoreFiles": [] +} +``` +### IgnoreFiles + +OPTIONAL List of file to ignore. diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/CompilerPlugin/CompilerPlugin.py b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/CompilerPlugin/CompilerPlugin.py new file mode 100755 index 00000000..a805ac33 --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/CompilerPlugin/CompilerPlugin.py @@ -0,0 +1,102 @@ +# @file CompilerPlugin.py +## +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +import logging +import os +import re +from edk2toollib.uefi.edk2.parsers.dsc_parser import DscParser +from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin +from edk2toolext.environment.uefi_build import UefiBuilder +from edk2toolext import edk2_logging +from edk2toolext.environment.var_dict import VarDict + + +class CompilerPlugin(ICiBuildPlugin): + """ + A CiBuildPlugin that compiles the package dsc + from the package being tested. + + Configuration options: + "CompilerPlugin": { + "DscPath": "<path to dsc from root of pkg>" + } + """ + + def GetTestName(self, packagename: str, environment: VarDict) -> tuple: + """ Provide the testcase name and classname for use in reporting + + Args: + packagename: string containing name of package to build + environment: The VarDict for the test to run in + Returns: + a tuple containing the testcase name and the classname + (testcasename, classname) + """ + target = environment.GetValue("TARGET") + return ("Compile " + packagename + " " + target, packagename + ".Compiler." + target) + + def RunsOnTargetList(self): + return ["DEBUG", "RELEASE"] + + ## + # External function of plugin. This function is used to perform the task of the ICiBuildPlugin Plugin + # + # - package is the edk2 path to package. This means workspace/packagepath relative. + # - edk2path object configured with workspace and packages path + # - PkgConfig Object (dict) for the pkg + # - EnvConfig Object + # - Plugin Manager Instance + # - Plugin Helper Obj Instance + # - Junit Logger + # - output_stream the StringIO output stream from this plugin via logging + def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None): + self._env = environment + + # Parse the config for required DscPath element + if "DscPath" not in pkgconfig: + tc.SetSkipped() + tc.LogStdError("DscPath not found in config file. Nothing to compile.") + return -1 + + AP = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagename) + + APDSC = os.path.join(AP, pkgconfig["DscPath"].strip()) + AP_Path = Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(APDSC) + if AP is None or AP_Path is None or not os.path.isfile(APDSC): + tc.SetSkipped() + tc.LogStdError("Package Dsc not found.") + return -1 + + logging.info("Building {0}".format(AP_Path)) + self._env.SetValue("ACTIVE_PLATFORM", AP_Path, "Set in Compiler Plugin") + + # Parse DSC to check for SUPPORTED_ARCHITECTURES + dp = DscParser() + dp.SetBaseAbsPath(Edk2pathObj.WorkspacePath) + dp.SetPackagePaths(Edk2pathObj.PackagePathList) + dp.ParseFile(AP_Path) + if "SUPPORTED_ARCHITECTURES" in dp.LocalVars: + SUPPORTED_ARCHITECTURES = dp.LocalVars["SUPPORTED_ARCHITECTURES"].split('|') + TARGET_ARCHITECTURES = environment.GetValue("TARGET_ARCH").split(' ') + + # Skip if there is no intersection between SUPPORTED_ARCHITECTURES and TARGET_ARCHITECTURES + if len(set(SUPPORTED_ARCHITECTURES) & set(TARGET_ARCHITECTURES)) == 0: + tc.SetSkipped() + tc.LogStdError("No supported architecutres to build") + return -1 + + uefiBuilder = UefiBuilder() + # do all the steps + # WorkSpace, PackagesPath, PInHelper, PInManager + ret = uefiBuilder.Go(Edk2pathObj.WorkspacePath, os.pathsep.join(Edk2pathObj.PackagePathList), PLMHelper, PLM) + if ret != 0: # failure: + tc.SetFailed("Compile failed for {0}".format(packagename), "Compile_FAILED") + tc.LogStdError("{0} Compile failed with error code {1} ".format(AP_Path, ret)) + return 1 + + else: + tc.SetSuccess() + return 0 diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/CompilerPlugin/Compiler_plug_in.yaml b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/CompilerPlugin/Compiler_plug_in.yaml new file mode 100644 index 00000000..09a41dd1 --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/CompilerPlugin/Compiler_plug_in.yaml @@ -0,0 +1,11 @@ +## @file +# CiBuildPlugin used to compile each package +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "cibuild", + "name": "Compiler Plugin", + "module": "CompilerPlugin" +} diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/CompilerPlugin/Readme.md b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/CompilerPlugin/Readme.md new file mode 100644 index 00000000..09352ed8 --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/CompilerPlugin/Readme.md @@ -0,0 +1,17 @@ +# Compiler Plugin + +This CiBuildPlugin compiles the package DSC from the package being tested. + +## Configuration + +The package relative path of the DSC file to build. + +``` yaml +"CompilerPlugin": { + "DscPath": "<path to dsc from root of pkg>" +} +``` + +### DscPath + +Package relative path to the DSC file to build. diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/DependencyCheck/DependencyCheck.py b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/DependencyCheck/DependencyCheck.py new file mode 100755 index 00000000..47f03336 --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/DependencyCheck/DependencyCheck.py @@ -0,0 +1,120 @@ +# @file dependency_check.py +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +import logging +import os +from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin +from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser +from edk2toolext.environment.var_dict import VarDict + + +class DependencyCheck(ICiBuildPlugin): + """ + A CiBuildPlugin that finds all modules (inf files) in a package and reviews the packages used + to confirm they are acceptable. This is to help enforce layering and identify improper + dependencies between packages. + + Configuration options: + "DependencyCheck": { + "AcceptableDependencies": [], # Package dec files that are allowed in all INFs. Example: MdePkg/MdePkg.dec + "AcceptableDependencies-<MODULE_TYPE>": [], # OPTIONAL Package dependencies for INFs that are HOST_APPLICATION + "AcceptableDependencies-HOST_APPLICATION": [], # EXAMPLE Package dependencies for INFs that are HOST_APPLICATION + "IgnoreInf": [] # Ignore INF if found in filesystem + } + """ + + def GetTestName(self, packagename: str, environment: VarDict) -> tuple: + """ Provide the testcase name and classname for use in reporting + + Args: + packagename: string containing name of package to build + environment: The VarDict for the test to run in + Returns: + a tuple containing the testcase name and the classname + (testcasename, classname) + testclassname: a descriptive string for the testcase can include whitespace + classname: should be patterned <packagename>.<plugin>.<optionally any unique condition> + """ + return ("Test Package Dependencies for modules in " + packagename, packagename + ".DependencyCheck") + + ## + # External function of plugin. This function is used to perform the task of the MuBuild Plugin + # + # - package is the edk2 path to package. This means workspace/packagepath relative. + # - edk2path object configured with workspace and packages path + # - PkgConfig Object (dict) for the pkg + # - EnvConfig Object + # - Plugin Manager Instance + # - Plugin Helper Obj Instance + # - Junit Logger + # - output_stream the StringIO output stream from this plugin via logging + def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None): + overall_status = 0 + + # Get current platform + abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagename) + + # Get INF Files + INFFiles = self.WalkDirectoryForExtension([".inf"], abs_pkg_path) + INFFiles = [Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(x) for x in INFFiles] # make edk2relative path so can compare with Ignore List + + # Remove ignored INFs + if "IgnoreInf" in pkgconfig: + for a in pkgconfig["IgnoreInf"]: + a = a.replace(os.sep, "/") ## convert path sep in case ignore list is bad. Can't change case + try: + INFFiles.remove(a) + tc.LogStdOut("IgnoreInf {0}".format(a)) + except: + logging.info("DependencyConfig.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a)) + tc.LogStdError("DependencyConfig.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a)) + + + # Get the AccpetableDependencies list + if "AcceptableDependencies" not in pkgconfig: + logging.info("DependencyCheck Skipped. No Acceptable Dependencies defined.") + tc.LogStdOut("DependencyCheck Skipped. No Acceptable Dependencies defined.") + tc.SetSkipped() + return -1 + + # Log dependencies + for k in pkgconfig.keys(): + if k.startswith("AcceptableDependencies"): + pkgstring = "\n".join(pkgconfig[k]) + if ("-" in k): + _, _, mod_type = k.partition("-") + tc.LogStdOut(f"Additional dependencies for MODULE_TYPE {mod_type}:\n {pkgstring}") + else: + tc.LogStdOut(f"Acceptable Dependencies:\n {pkgstring}") + + # For each INF file + for file in INFFiles: + ip = InfParser() + logging.debug("Parsing " + file) + ip.SetBaseAbsPath(Edk2pathObj.WorkspacePath).SetPackagePaths(Edk2pathObj.PackagePathList).ParseFile(file) + + if("MODULE_TYPE" not in ip.Dict): + tc.LogStdOut("Ignoring INF. Missing key for MODULE_TYPE {0}".format(file)) + continue + + mod_type = ip.Dict["MODULE_TYPE"].upper() + for p in ip.PackagesUsed: + if p not in pkgconfig["AcceptableDependencies"]: + # If not in the main acceptable dependencies list then check module specific + mod_specific_key = "AcceptableDependencies-" + mod_type + if mod_specific_key in pkgconfig and p in pkgconfig[mod_specific_key]: + continue + + logging.error("Dependency Check: Invalid Dependency INF: {0} depends on pkg {1}".format(file, p)) + tc.LogStdError("Dependency Check: Invalid Dependency INF: {0} depends on pkg {1}".format(file, p)) + overall_status += 1 + + # If XML object exists, add results + if overall_status != 0: + tc.SetFailed("Failed with {0} errors".format(overall_status), "DEPENDENCYCHECK_FAILED") + else: + tc.SetSuccess() + return overall_status diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/DependencyCheck/DependencyCheck_plug_in.yaml b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/DependencyCheck/DependencyCheck_plug_in.yaml new file mode 100644 index 00000000..1bd1ce6d --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/DependencyCheck/DependencyCheck_plug_in.yaml @@ -0,0 +1,13 @@ +## @file +# CiBuildPlugin used to check all infs within a package +# to confirm the packagesdependency are on the configured list of acceptable +# dependencies. +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "cibuild", + "name": "Dependency Check Test", + "module": "DependencyCheck" +} diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/DependencyCheck/Readme.md b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/DependencyCheck/Readme.md new file mode 100644 index 00000000..a5bb2d5c --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/DependencyCheck/Readme.md @@ -0,0 +1,31 @@ +# Depdendency Check Plugin + +A CiBuildPlugin that finds all modules (inf files) in a package and reviews the +packages used to confirm they are acceptable. This is to help enforce layering +and identify improper dependencies between packages. + +## Configuration + +The plugin must be configured with the acceptabe package dependencies for the +package. + +``` yaml +"DependencyCheck": { + "AcceptableDependencies": [], + "AcceptableDependencies-<MODULE_TYPE>": [], + "IgnoreInf": [] +} +``` + +### AcceptableDependencies + +Package dec files that are allowed in all INFs. Example: MdePkg/MdePkg.dec + +### AcceptableDependencies-<MODULE_TYPE> + +OPTIONAL Package dependencies for INFs that have module type <MODULE_TYPE>. +Example: AcceptableDependencies-HOST_APPLICATION. + +### IgnoreInf + +OPTIONAL list of INFs to ignore for this dependency check. diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/DscCompleteCheck/DscCompleteCheck.py b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/DscCompleteCheck/DscCompleteCheck.py new file mode 100755 index 00000000..8113ddbe --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/DscCompleteCheck/DscCompleteCheck.py @@ -0,0 +1,133 @@ +# @file DscCompleteCheck.py +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +import logging +import os +from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin +from edk2toollib.uefi.edk2.parsers.dsc_parser import DscParser +from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser +from edk2toolext.environment.var_dict import VarDict + + +class DscCompleteCheck(ICiBuildPlugin): + """ + A CiBuildPlugin that scans the package dsc file and confirms all modules (inf files) are + listed in the components sections. + + Configuration options: + "DscCompleteCheck": { + "DscPath": "<path to dsc from root of pkg>" + "IgnoreInf": [] # Ignore INF if found in filesystem by not dsc + } + """ + + def GetTestName(self, packagename: str, environment: VarDict) -> tuple: + """ Provide the testcase name and classname for use in reporting + + Args: + packagename: string containing name of package to build + environment: The VarDict for the test to run in + Returns: + a tuple containing the testcase name and the classname + (testcasename, classname) + testclassname: a descriptive string for the testcase can include whitespace + classname: should be patterned <packagename>.<plugin>.<optionally any unique condition> + """ + return ("Check the " + packagename + " DSC for a being complete", packagename + ".DscCompleteCheck") + + ## + # External function of plugin. This function is used to perform the task of the MuBuild Plugin + # + # - package is the edk2 path to package. This means workspace/packagepath relative. + # - edk2path object configured with workspace and packages path + # - PkgConfig Object (dict) for the pkg + # - VarDict containing the shell environment Build Vars + # - Plugin Manager Instance + # - Plugin Helper Obj Instance + # - Junit Logger + # - output_stream the StringIO output stream from this plugin via logging + def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None): + overall_status = 0 + + # Parse the config for required DscPath element + if "DscPath" not in pkgconfig: + tc.SetSkipped() + tc.LogStdError( + "DscPath not found in config file. Nothing to check.") + return -1 + + abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath( + packagename) + abs_dsc_path = os.path.join(abs_pkg_path, pkgconfig["DscPath"].strip()) + wsr_dsc_path = Edk2pathObj.GetEdk2RelativePathFromAbsolutePath( + abs_dsc_path) + + if abs_dsc_path is None or wsr_dsc_path == "" or not os.path.isfile(abs_dsc_path): + tc.SetSkipped() + tc.LogStdError("Package Dsc not found") + return 0 + + # Get INF Files + INFFiles = self.WalkDirectoryForExtension([".inf"], abs_pkg_path) + INFFiles = [Edk2pathObj.GetEdk2RelativePathFromAbsolutePath( + x) for x in INFFiles] # make edk2relative path so can compare with DSC + + # remove ignores + + if "IgnoreInf" in pkgconfig: + for a in pkgconfig["IgnoreInf"]: + a = a.replace(os.sep, "/") + try: + tc.LogStdOut("Ignoring INF {0}".format(a)) + INFFiles.remove(a) + except: + tc.LogStdError( + "DscCompleteCheck.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a)) + logging.info( + "DscCompleteCheck.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a)) + + # DSC Parser + dp = DscParser() + dp.SetBaseAbsPath(Edk2pathObj.WorkspacePath) + dp.SetPackagePaths(Edk2pathObj.PackagePathList) + dp.SetInputVars(environment.GetAllBuildKeyValues()) + dp.ParseFile(wsr_dsc_path) + + # Check if INF in component section + for INF in INFFiles: + if not any(INF.strip() in x for x in dp.ThreeMods) and \ + not any(INF.strip() in x for x in dp.SixMods) and \ + not any(INF.strip() in x for x in dp.OtherMods): + + infp = InfParser().SetBaseAbsPath(Edk2pathObj.WorkspacePath) + infp.SetPackagePaths(Edk2pathObj.PackagePathList) + infp.ParseFile(INF) + if("MODULE_TYPE" not in infp.Dict): + tc.LogStdOut( + "Ignoring INF. Missing key for MODULE_TYPE {0}".format(INF)) + continue + + if(infp.Dict["MODULE_TYPE"] == "HOST_APPLICATION"): + tc.LogStdOut( + "Ignoring INF. Module type is HOST_APPLICATION {0}".format(INF)) + continue + + if len(infp.SupportedPhases) == 1 and \ + "HOST_APPLICATION" in infp.SupportedPhases: + tc.LogStdOut( + "Ignoring Library INF due to only supporting type HOST_APPLICATION {0}".format(INF)) + continue + + logging.critical(INF + " not in " + wsr_dsc_path) + tc.LogStdError("{0} not in {1}".format(INF, wsr_dsc_path)) + overall_status = overall_status + 1 + + # If XML object exists, add result + if overall_status != 0: + tc.SetFailed("DscCompleteCheck {0} Failed. Errors {1}".format( + wsr_dsc_path, overall_status), "CHECK_FAILED") + else: + tc.SetSuccess() + return overall_status diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/DscCompleteCheck/DscCompleteCheck_plug_in.yaml b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/DscCompleteCheck/DscCompleteCheck_plug_in.yaml new file mode 100644 index 00000000..a8035abb --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/DscCompleteCheck/DscCompleteCheck_plug_in.yaml @@ -0,0 +1,12 @@ +## @file +# CiBuildPlugin used to confirm all INFs are listed in +# the components section of package dsc +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "cibuild", + "name": "Dsc Complete Check Test", + "module": "DscCompleteCheck" +} diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/DscCompleteCheck/Readme.md b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/DscCompleteCheck/Readme.md new file mode 100644 index 00000000..8f85fda0 --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/DscCompleteCheck/Readme.md @@ -0,0 +1,32 @@ +# Dsc Complete Check Plugin + +This CiBuildPlugin scans all INF files from a package and confirms they are +listed in the package level DSC file. The test considers it an error if any INF +does not appear in the `Components` section of the package-level DSC (indicating +that it would not be built if the package were built). This is critical because +much of the CI infrastructure assumes that all modules will be listed in the DSC +and compiled. + +This test will ignore INFs in the following cases: + +1. When MODULE_TYPE = HOST_APPLICATION +2. When a Library instance **only** supports the HOST_APPLICATION environment + +## Configuration + +The plugin has a few configuration options to support the UEFI codebase. + +``` yaml +"DscCompleteCheck": { + "DscPath": "", # Path to dsc from root of package + "IgnoreInf": [] # Ignore INF if found in filesystem but not dsc + } +``` + +### DscPath + +Path to DSC to consider platform dsc + +### IgnoreInf + +Ignore error if Inf file is not listed in DSC file diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/EccCheck/EccCheck.py b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/EccCheck/EccCheck.py new file mode 100755 index 00000000..601cf3d2 --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/EccCheck/EccCheck.py @@ -0,0 +1,309 @@ +# @file EccCheck.py +# +# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR> +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +import os +import shutil +import re +import csv +import xml.dom.minidom +from typing import List, Dict, Tuple +import logging +from io import StringIO +from edk2toolext.environment import shell_environment +from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin +from edk2toolext.environment.var_dict import VarDict +from edk2toollib.utility_functions import RunCmd + + +class EccCheck(ICiBuildPlugin): + """ + A CiBuildPlugin that finds the Ecc issues of newly added code in pull request. + + Configuration options: + "EccCheck": { + "ExceptionList": [], + "IgnoreFiles": [] + }, + """ + + ReModifyFile = re.compile(r'[B-Q,S-Z]+[\d]*\t(.*)') + FindModifyFile = re.compile(r'\+\+\+ b\/(.*)') + LineScopePattern = (r'@@ -\d*\,*\d* \+\d*\,*\d* @@.*') + LineNumRange = re.compile(r'@@ -\d*\,*\d* \+(\d*)\,*(\d*) @@.*') + + def GetTestName(self, packagename: str, environment: VarDict) -> tuple: + """ Provide the testcase name and classname for use in reporting + testclassname: a descriptive string for the testcase can include whitespace + classname: should be patterned <packagename>.<plugin>.<optionally any unique condition> + + Args: + packagename: string containing name of package to build + environment: The VarDict for the test to run in + Returns: + a tuple containing the testcase name and the classname + (testcasename, classname) + """ + return ("Check for efi coding style for " + packagename, packagename + ".EccCheck") + + ## + # External function of plugin. This function is used to perform the task of the ci_build_plugin Plugin + # + # - package is the edk2 path to package. This means workspace/packagepath relative. + # - edk2path object configured with workspace and packages path + # - PkgConfig Object (dict) for the pkg + # - EnvConfig Object + # - Plugin Manager Instance + # - Plugin Helper Obj Instance + # - Junit Logger + # - output_stream the StringIO output stream from this plugin via logging + def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None): + edk2_path = Edk2pathObj.WorkspacePath + python_path = os.path.join(edk2_path, "BaseTools", "Source", "Python") + env = shell_environment.GetEnvironment() + env.set_shell_var('PYTHONPATH', python_path) + env.set_shell_var('WORKSPACE', edk2_path) + self.ECC_PASS = True + self.ApplyConfig(pkgconfig, edk2_path, packagename) + modify_dir_list = self.GetModifyDir(packagename) + patch = self.GetDiff(packagename) + ecc_diff_range = self.GetDiffRange(patch, packagename, edk2_path) + self.GenerateEccReport(modify_dir_list, ecc_diff_range, edk2_path) + ecc_log = os.path.join(edk2_path, "Ecc.log") + self.RevertCode() + if self.ECC_PASS: + tc.SetSuccess() + self.RemoveFile(ecc_log) + return 0 + else: + with open(ecc_log, encoding='utf8') as output: + ecc_output = output.readlines() + for line in ecc_output: + logging.error(line.strip()) + self.RemoveFile(ecc_log) + tc.SetFailed("EccCheck failed for {0}".format(packagename), "Ecc detected issues") + return 1 + + def RevertCode(self) -> None: + submoudle_params = "submodule update --init" + RunCmd("git", submoudle_params) + reset_params = "reset HEAD --hard" + RunCmd("git", reset_params) + + def GetDiff(self, pkg: str) -> List[str]: + return_buffer = StringIO() + params = "diff --unified=0 origin/master HEAD" + RunCmd("git", params, outstream=return_buffer) + p = return_buffer.getvalue().strip() + patch = p.split("\n") + return_buffer.close() + + return patch + + def RemoveFile(self, file: str) -> None: + if os.path.exists(file): + os.remove(file) + return + + def GetModifyDir(self, pkg: str) -> List[str]: + return_buffer = StringIO() + params = "diff --name-status" + ' HEAD' + ' origin/master' + RunCmd("git", params, outstream=return_buffer) + p1 = return_buffer.getvalue().strip() + dir_list = p1.split("\n") + return_buffer.close() + modify_dir_list = [] + for modify_dir in dir_list: + file_path = self.ReModifyFile.findall(modify_dir) + if file_path: + file_dir = os.path.dirname(file_path[0]) + else: + continue + if pkg in file_dir and file_dir != pkg: + modify_dir_list.append('%s' % file_dir) + else: + continue + + modify_dir_list = list(set(modify_dir_list)) + return modify_dir_list + + def GetDiffRange(self, patch_diff: List[str], pkg: str, workingdir: str) -> Dict[str, List[Tuple[int, int]]]: + IsDelete = True + StartCheck = False + range_directory: Dict[str, List[Tuple[int, int]]] = {} + for line in patch_diff: + modify_file = self.FindModifyFile.findall(line) + if modify_file and pkg in modify_file[0] and not StartCheck and os.path.isfile(modify_file[0]): + modify_file_comment_dic = self.GetCommentRange(modify_file[0], workingdir) + IsDelete = False + StartCheck = True + modify_file_dic = modify_file[0] + modify_file_dic = modify_file_dic.replace("/", os.sep) + range_directory[modify_file_dic] = [] + elif line.startswith('--- '): + StartCheck = False + elif re.match(self.LineScopePattern, line, re.I) and not IsDelete and StartCheck: + start_line = self.LineNumRange.search(line).group(1) + line_range = self.LineNumRange.search(line).group(2) + if not line_range: + line_range = '1' + range_directory[modify_file_dic].append((int(start_line), int(start_line) + int(line_range) - 1)) + for i in modify_file_comment_dic: + if int(i[0]) <= int(start_line) <= int(i[1]): + range_directory[modify_file_dic].append(i) + return range_directory + + def GetCommentRange(self, modify_file: str, workingdir: str) -> List[Tuple[int, int]]: + modify_file_path = os.path.join(workingdir, modify_file) + with open(modify_file_path) as f: + line_no = 1 + comment_range: List[Tuple[int, int]] = [] + Start = False + for line in f: + if line.startswith('/**'): + start_no = line_no + Start = True + if line.startswith('**/') and Start: + end_no = line_no + Start = False + comment_range.append((int(start_no), int(end_no))) + line_no += 1 + + if comment_range and comment_range[0][0] == 1: + del comment_range[0] + return comment_range + + def GenerateEccReport(self, modify_dir_list: List[str], ecc_diff_range: Dict[str, List[Tuple[int, int]]], + edk2_path: str) -> None: + ecc_need = False + ecc_run = True + config = os.path.join(edk2_path, "BaseTools", "Source", "Python", "Ecc", "config.ini") + exception = os.path.join(edk2_path, "BaseTools", "Source", "Python", "Ecc", "exception.xml") + report = os.path.join(edk2_path, "Ecc.csv") + for modify_dir in modify_dir_list: + target = os.path.join(edk2_path, modify_dir) + logging.info('Run ECC tool for the commit in %s' % modify_dir) + ecc_need = True + ecc_params = "-c {0} -e {1} -t {2} -r {3}".format(config, exception, target, report) + return_code = RunCmd("Ecc", ecc_params, workingdir=edk2_path) + if return_code != 0: + ecc_run = False + break + if not ecc_run: + logging.error('Fail to run ECC tool') + self.ParseEccReport(ecc_diff_range, edk2_path) + + if not ecc_need: + logging.info("Doesn't need run ECC check") + + revert_params = "checkout -- {}".format(exception) + RunCmd("git", revert_params) + return + + def ParseEccReport(self, ecc_diff_range: Dict[str, List[Tuple[int, int]]], edk2_path: str) -> None: + ecc_log = os.path.join(edk2_path, "Ecc.log") + ecc_csv = "Ecc.csv" + file = os.listdir(edk2_path) + row_lines = [] + ignore_error_code = self.GetIgnoreErrorCode() + if ecc_csv in file: + with open(ecc_csv) as csv_file: + reader = csv.reader(csv_file) + for row in reader: + for modify_file in ecc_diff_range: + if modify_file in row[3]: + for i in ecc_diff_range[modify_file]: + line_no = int(row[4]) + if i[0] <= line_no <= i[1] and row[1] not in ignore_error_code: + row[0] = '\nEFI coding style error' + row[1] = 'Error code: ' + row[1] + row[3] = 'file: ' + row[3] + row[4] = 'Line number: ' + row[4] + row_line = '\n *'.join(row) + row_lines.append(row_line) + break + break + if row_lines: + self.ECC_PASS = False + + with open(ecc_log, 'a') as log: + all_line = '\n'.join(row_lines) + all_line = all_line + '\n' + log.writelines(all_line) + return + + def ApplyConfig(self, pkgconfig: Dict[str, List[str]], edk2_path: str, pkg: str) -> None: + if "IgnoreFiles" in pkgconfig: + for a in pkgconfig["IgnoreFiles"]: + a = os.path.join(edk2_path, pkg, a) + a = a.replace(os.sep, "/") + + logging.info("Ignoring Files {0}".format(a)) + if os.path.exists(a): + if os.path.isfile(a): + self.RemoveFile(a) + elif os.path.isdir(a): + shutil.rmtree(a) + else: + logging.error("EccCheck.IgnoreInf -> {0} not found in filesystem. Invalid ignore files".format(a)) + + if "ExceptionList" in pkgconfig: + exception_list = pkgconfig["ExceptionList"] + exception_xml = os.path.join(edk2_path, "BaseTools", "Source", "Python", "Ecc", "exception.xml") + try: + logging.info("Appending exceptions") + self.AppendException(exception_list, exception_xml) + except Exception as e: + logging.error("Fail to apply exceptions") + raise e + return + + def AppendException(self, exception_list: List[str], exception_xml: str) -> None: + error_code_list = exception_list[::2] + keyword_list = exception_list[1::2] + dom_tree = xml.dom.minidom.parse(exception_xml) + root_node = dom_tree.documentElement + for error_code, keyword in zip(error_code_list, keyword_list): + customer_node = dom_tree.createElement("Exception") + keyword_node = dom_tree.createElement("KeyWord") + keyword_node_text_value = dom_tree.createTextNode(keyword) + keyword_node.appendChild(keyword_node_text_value) + customer_node.appendChild(keyword_node) + error_code_node = dom_tree.createElement("ErrorID") + error_code_text_value = dom_tree.createTextNode(error_code) + error_code_node.appendChild(error_code_text_value) + customer_node.appendChild(error_code_node) + root_node.appendChild(customer_node) + with open(exception_xml, 'w') as f: + dom_tree.writexml(f, indent='', addindent='', newl='\n', encoding='UTF-8') + return + + def GetIgnoreErrorCode(self) -> set: + """ + Below are kinds of error code that are accurate in ecc scanning of edk2 level. + But EccCheck plugin is partial scanning so they are always false positive issues. + The mapping relationship of error code and error message is listed BaseTools/Sourc/Python/Ecc/EccToolError.py + """ + ignore_error_code = { + "10000", + "10001", + "10002", + "10003", + "10004", + "10005", + "10006", + "10007", + "10008", + "10009", + "10010", + "10011", + "10012", + "10013", + "10015", + "10016", + "10017", + "10022", + } + return ignore_error_code diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/EccCheck/EccCheck_plug_in.yaml b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/EccCheck/EccCheck_plug_in.yaml new file mode 100644 index 00000000..bc3ea6e2 --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/EccCheck/EccCheck_plug_in.yaml @@ -0,0 +1,11 @@ +## @file +# CiBuildPlugin used to check Ecc issues +# +# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR> +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "cibuild", + "name": "EccCheck Test", + "module": "EccCheck" +} diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/EccCheck/Readme.md b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/EccCheck/Readme.md new file mode 100644 index 00000000..02e0c130 --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/EccCheck/Readme.md @@ -0,0 +1,15 @@ +# EFI Coding style Check Plugin + +This CiBuildPlugin finds the Ecc issues of newly added code in pull request. + +## Configuration + +The plugin can be configured to ignore certain files and issues. + +"EccCheck": { + "ExceptionList": [], + "IgnoreFiles": [] + }, + """ + +OPTIONAL List of file to ignore. diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/GuidCheck/GuidCheck.py b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/GuidCheck/GuidCheck.py new file mode 100755 index 00000000..73fdf66d --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/GuidCheck/GuidCheck.py @@ -0,0 +1,251 @@ +# @file GuidCheck.py +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +import logging +from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin +from edk2toollib.uefi.edk2.guid_list import GuidList +from edk2toolext.environment.var_dict import VarDict + + +class GuidCheck(ICiBuildPlugin): + """ + A CiBuildPlugin that scans the code tree and looks for duplicate guids + from the package being tested. + + Configuration options: + "GuidCheck": { + "IgnoreGuidName": [], # provide in format guidname=guidvalue or just guidname + "IgnoreGuidValue": [], + "IgnoreFoldersAndFiles": [], + "IgnoreDuplicates": [] # Provide in format guidname=guidname=guidname... + } + """ + + def GetTestName(self, packagename: str, environment: VarDict) -> tuple: + """ Provide the testcase name and classname for use in reporting + + Args: + packagename: string containing name of package to build + environment: The VarDict for the test to run in + Returns: + a tuple containing the testcase name and the classname + (testcasename, classname) + testclassname: a descriptive string for the testcase can include whitespace + classname: should be patterned <packagename>.<plugin>.<optionally any unique condition> + """ + return ("Confirm GUIDs are unique in " + packagename, packagename + ".GuidCheck") + + def _FindConflictingGuidValues(self, guidlist: list) -> list: + """ Find all duplicate guids by guid value and report them as errors + """ + # Sort the list by guid + guidsorted = sorted( + guidlist, key=lambda x: x.guid.upper(), reverse=True) + + previous = None # Store previous entry for comparison + error = None + errors = [] + for index in range(len(guidsorted)): + i = guidsorted[index] + if(previous is not None): + if i.guid == previous.guid: # Error + if(error is None): + # Catch errors with more than 1 conflict + error = ErrorEntry("guid") + error.entries.append(previous) + errors.append(error) + error.entries.append(i) + else: + # no match. clear error + error = None + previous = i + return errors + + def _FindConflictingGuidNames(self, guidlist: list) -> list: + """ Find all duplicate guids by name and if they are not all + from inf files report them as errors. It is ok to have + BASE_NAME duplication. + + Is this useful? It would catch two same named guids in dec file + that resolve to different values. + """ + # Sort the list by guid + namesorted = sorted(guidlist, key=lambda x: x.name.upper()) + + previous = None # Store previous entry for comparison + error = None + errors = [] + for index in range(len(namesorted)): + i = namesorted[index] + if(previous is not None): + # If name matches + if i.name == previous.name: + if(error is None): + # Catch errors with more than 1 conflict + error = ErrorEntry("name") + error.entries.append(previous) + errors.append(error) + error.entries.append(i) + else: + # no match. clear error + error = None + previous = i + + # Loop thru and remove any errors where all files are infs as it is ok if + # they have the same inf base name. + for e in errors[:]: + if len( [en for en in e.entries if not en.absfilepath.lower().endswith(".inf")]) == 0: + errors.remove(e) + + return errors + + ## + # External function of plugin. This function is used to perform the task of the MuBuild Plugin + # + # - package is the edk2 path to package. This means workspace/packagepath relative. + # - edk2path object configured with workspace and packages path + # - PkgConfig Object (dict) for the pkg + # - EnvConfig Object + # - Plugin Manager Instance + # - Plugin Helper Obj Instance + # - Junit Logger + # - output_stream the StringIO output stream from this plugin via logging + + def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None): + Errors = [] + + abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath( + packagename) + + if abs_pkg_path is None: + tc.SetSkipped() + tc.LogStdError("No package {0}".format(packagename)) + return -1 + + All_Ignores = ["/Build", "/Conf"] + # Parse the config for other ignores + if "IgnoreFoldersAndFiles" in pkgconfig: + All_Ignores.extend(pkgconfig["IgnoreFoldersAndFiles"]) + + # Parse the workspace for all GUIDs + gs = GuidList.guidlist_from_filesystem( + Edk2pathObj.WorkspacePath, ignore_lines=All_Ignores) + + # Remove ignored guidvalue + if "IgnoreGuidValue" in pkgconfig: + for a in pkgconfig["IgnoreGuidValue"]: + try: + tc.LogStdOut("Ignoring Guid {0}".format(a.upper())) + for b in gs[:]: + if b.guid == a.upper(): + gs.remove(b) + except: + tc.LogStdError("GuidCheck.IgnoreGuid -> {0} not found. Invalid ignore guid".format(a.upper())) + logging.info("GuidCheck.IgnoreGuid -> {0} not found. Invalid ignore guid".format(a.upper())) + + # Remove ignored guidname + if "IgnoreGuidName" in pkgconfig: + for a in pkgconfig["IgnoreGuidName"]: + entry = a.split("=") + if(len(entry) > 2): + tc.LogStdError("GuidCheck.IgnoreGuidName -> {0} Invalid Format.".format(a)) + logging.info("GuidCheck.IgnoreGuidName -> {0} Invalid Format.".format(a)) + continue + try: + tc.LogStdOut("Ignoring Guid {0}".format(a)) + for b in gs[:]: + if b.name == entry[0]: + if(len(entry) == 1): + gs.remove(b) + elif(len(entry) == 2 and b.guid.upper() == entry[1].upper()): + gs.remove(b) + else: + c.LogStdError("GuidCheck.IgnoreGuidName -> {0} incomplete match. Invalid ignore guid".format(a)) + + except: + tc.LogStdError("GuidCheck.IgnoreGuidName -> {0} not found. Invalid ignore name".format(a)) + logging.info("GuidCheck.IgnoreGuidName -> {0} not found. Invalid ignore name".format(a)) + + # Find conflicting Guid Values + Errors.extend(self._FindConflictingGuidValues(gs)) + + # Check if there are expected duplicates and remove it from the error list + if "IgnoreDuplicates" in pkgconfig: + for a in pkgconfig["IgnoreDuplicates"]: + names = a.split("=") + if len(names) < 2: + tc.LogStdError("GuidCheck.IgnoreDuplicates -> {0} invalid format".format(a)) + logging.info("GuidCheck.IgnoreDuplicates -> {0} invalid format".format(a)) + continue + + for b in Errors[:]: + if b.type != "guid": + continue + ## Make a list of the names that are not in the names list. If there + ## are any in the list then this error should not be ignored. + t = [x for x in b.entries if x.name not in names] + if(len(t) == len(b.entries)): + ## did not apply to any entry + continue + elif(len(t) == 0): + ## full match - ignore duplicate + tc.LogStdOut("GuidCheck.IgnoreDuplicates -> {0}".format(a)) + Errors.remove(b) + elif(len(t) < len(b.entries)): + ## partial match + tc.LogStdOut("GuidCheck.IgnoreDuplicates -> {0} incomplete match".format(a)) + logging.info("GuidCheck.IgnoreDuplicates -> {0} incomplete match".format(a)) + else: + tc.LogStdOut("GuidCheck.IgnoreDuplicates -> {0} unknown error.".format(a)) + logging.info("GuidCheck.IgnoreDuplicates -> {0} unknown error".format(a)) + + + + # Find conflicting Guid Names + Errors.extend(self._FindConflictingGuidNames(gs)) + + # Log errors for anything within the package under test + for er in Errors[:]: + InMyPackage = False + for a in er.entries: + if abs_pkg_path in a.absfilepath: + InMyPackage = True + break + if(not InMyPackage): + Errors.remove(er) + else: + logging.error(str(er)) + tc.LogStdError(str(er)) + + # add result to test case + overall_status = len(Errors) + if overall_status != 0: + tc.SetFailed("GuidCheck {0} Failed. Errors {1}".format( + packagename, overall_status), "CHECK_FAILED") + else: + tc.SetSuccess() + return overall_status + + +class ErrorEntry(): + """ Custom/private class for reporting errors in the GuidList + """ + + def __init__(self, errortype): + self.type = errortype # 'guid' or 'name' depending on error type + self.entries = [] # GuidListEntry that are in error condition + + def __str__(self): + a = f"Error Duplicate {self.type}: " + if(self.type == "guid"): + a += f" {self.entries[0].guid}" + elif(self.type == "name"): + a += f" {self.entries[0].name}" + + a += f" ({len(self.entries)})\n" + + for e in self.entries: + a += "\t" + str(e) + "\n" + return a diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/GuidCheck/GuidCheck_plug_in.yaml b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/GuidCheck/GuidCheck_plug_in.yaml new file mode 100644 index 00000000..dc40afad --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/GuidCheck/GuidCheck_plug_in.yaml @@ -0,0 +1,11 @@ +## @file +# CiBuildPlugin used to check guid uniqueness +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "cibuild", + "name": "Guid Check Test", + "module": "GuidCheck" +} diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/GuidCheck/Readme.md b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/GuidCheck/Readme.md new file mode 100644 index 00000000..36c63432 --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/GuidCheck/Readme.md @@ -0,0 +1,80 @@ +# Guid Check Plugin + +This CiBuildPlugin scans all the files in a code tree to find all the GUID +definitions. After collection it will then look for duplication in the package +under test. Uniqueness of all GUIDs are critical within the UEFI environment. +Duplication can cause numerous issues including locating the wrong data +structure, calling the wrong function, or decoding the wrong data members. + +Currently Scanned: + +* INF files are scanned for there Module guid +* DEC files are scanned for all of their Protocols, PPIs, and Guids as well as + the one package GUID. + +Any GUID value being equal to two names or even just defined in two files is +considered an error unless in the ignore list. + +Any GUID name that is found more than once is an error unless all occurrences +are Module GUIDs. Since the Module GUID is assigned to the Module name it is +common to have numerous versions of the same module named the same. + +## Configuration + +The plugin has numerous configuration options to support the UEFI codebase. + +``` yaml +"GuidCheck": { + "IgnoreGuidName": [], + "IgnoreGuidValue": [], + "IgnoreFoldersAndFiles": [], + "IgnoreDuplicates": [] + } +``` + +### IgnoreGuidName + +This list allows strings in two formats. + +* _GuidName_ + * This will remove any entry with this GuidName from the list of GUIDs + therefore ignoring any error associated with this name. +* _GuidName=GuidValue_ + * This will also ignore the GUID by name but only if the value equals the + GuidValue. + * GuidValue should be in registry format. + * This is the suggested format to use as it will limit the ignore to only the + defined case. + +### IgnoreGuidValue + +This list allows strings in guid registry format _GuidValue_. + +* This will remove any entry with this GuidValue from the list of GUIDs + therefore ignoring any error associated with this value. +* GuidValue must be in registry format xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + +### IgnoreFoldersAndFiles + +This supports .gitignore file and folder matching strings including wildcards + +* Any folder or file ignored will not be parsed and therefore any GUID defined + will be ignored. +* The plugin will always ignores the following ["/Build", "/Conf"] + +### IgnoreDuplicates + +This supports strings in the format of _GuidName_=_GuidName_=_GuidName_ + +* For the error with the GuidNames to be ignored the list must match completely + with what is found during the code scan. + * For example if there are two GUIDs that are by design equal within the code + tree then it should be _GuidName_=_GuidName_ + * If instead there are three GUIDs then it must be + _GuidName_=_GuidName_=_GuidName_ +* This is the best ignore list to use because it is the most strict and will + catch new problems when new conflicts are introduced. +* There are numerous places in the UEFI specification in which two GUID names + are assigned the same value. These names should be set in this ignore list so + that they don't cause an error but any additional duplication would still be + caught. diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/HostUnitTestCompilerPlugin/HostUnitTestCompilerPlugin.py b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/HostUnitTestCompilerPlugin/HostUnitTestCompilerPlugin.py new file mode 100755 index 00000000..eaf1beb2 --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/HostUnitTestCompilerPlugin/HostUnitTestCompilerPlugin.py @@ -0,0 +1,149 @@ +# @file HostUnitTestCompilerPlugin.py +## +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +import logging +import os +import re +from edk2toollib.uefi.edk2.parsers.dsc_parser import DscParser +from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin +from edk2toolext.environment.uefi_build import UefiBuilder +from edk2toolext import edk2_logging +from edk2toolext.environment.var_dict import VarDict +from edk2toollib.utility_functions import GetHostInfo + + +class HostUnitTestCompilerPlugin(ICiBuildPlugin): + """ + A CiBuildPlugin that compiles the dsc for host based unit test apps. + An IUefiBuildPlugin may be attached to this plugin that will run the + unit tests and collect the results after successful compilation. + + Configuration options: + "HostUnitTestCompilerPlugin": { + "DscPath": "<path to dsc from root of pkg>" + } + """ + + def GetTestName(self, packagename: str, environment: VarDict) -> tuple: + """ Provide the testcase name and classname for use in reporting + testclassname: a descriptive string for the testcase can include whitespace + classname: should be patterned <packagename>.<plugin>.<optionally any unique condition> + + Args: + packagename: string containing name of package to build + environment: The VarDict for the test to run in + Returns: + a tuple containing the testcase name and the classname + (testcasename, classname) + """ + num,types = self.__GetHostUnitTestArch(environment) + types = types.replace(" ", "_") + + return ("Compile and Run Host-Based UnitTests for " + packagename + " on arch " + types, + packagename + ".HostUnitTestCompiler." + types) + + def RunsOnTargetList(self): + return ["NOOPT"] + + # + # Find the intersection of application types that can run on this host + # and the TARGET_ARCH being build in this request. + # + # return tuple with (number of UEFI arch types, space separated string) + def __GetHostUnitTestArch(self, environment): + requested = environment.GetValue("TARGET_ARCH").split(' ') + host = [] + if GetHostInfo().arch == 'x86': + #assume 64bit can handle 64 and 32 + #assume 32bit can only handle 32 + ## change once IA32 issues resolved host.append("IA32") + if GetHostInfo().bit == '64': + host.append("X64") + elif GetHostInfo().arch == 'ARM': + if GetHostInfo().bit == '64': + host.append("AARCH64") + elif GetHostInfo().bit == '32': + host.append("ARM") + + willrun = set(requested) & set(host) + return (len(willrun), " ".join(willrun)) + + + ## + # External function of plugin. This function is used to perform the task of the ICiBuildPlugin Plugin + # + # - package is the edk2 path to package. This means workspace/packagepath relative. + # - edk2path object configured with workspace and packages path + # - PkgConfig Object (dict) for the pkg + # - EnvConfig Object + # - Plugin Manager Instance + # - Plugin Helper Obj Instance + # - Junit Logger + # - output_stream the StringIO output stream from this plugin via logging + def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None): + self._env = environment + environment.SetValue("CI_BUILD_TYPE", "host_unit_test", "Set in HostUnitTestCompilerPlugin") + + # Parse the config for required DscPath element + if "DscPath" not in pkgconfig: + tc.SetSkipped() + tc.LogStdError("DscPath not found in config file. Nothing to compile for HostBasedUnitTests.") + return -1 + + AP = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagename) + + APDSC = os.path.join(AP, pkgconfig["DscPath"].strip()) + AP_Path = Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(APDSC) + if AP is None or AP_Path is None or not os.path.isfile(APDSC): + tc.SetSkipped() + tc.LogStdError("Package HostBasedUnitTest Dsc not found.") + return -1 + + logging.info("Building {0}".format(AP_Path)) + self._env.SetValue("ACTIVE_PLATFORM", AP_Path, "Set in Compiler Plugin") + num, RUNNABLE_ARCHITECTURES = self.__GetHostUnitTestArch(environment) + if(num == 0): + tc.SetSkipped() + tc.LogStdError("No host architecture compatibility") + return -1 + + if not environment.SetValue("TARGET_ARCH", + RUNNABLE_ARCHITECTURES, + "Update Target Arch based on Host Support"): + #use AllowOverride function since this is a controlled attempt to change + environment.AllowOverride("TARGET_ARCH") + if not environment.SetValue("TARGET_ARCH", + RUNNABLE_ARCHITECTURES, + "Update Target Arch based on Host Support"): + raise RuntimeError("Can't Change TARGET_ARCH as required") + + # Parse DSC to check for SUPPORTED_ARCHITECTURES + dp = DscParser() + dp.SetBaseAbsPath(Edk2pathObj.WorkspacePath) + dp.SetPackagePaths(Edk2pathObj.PackagePathList) + dp.ParseFile(AP_Path) + if "SUPPORTED_ARCHITECTURES" in dp.LocalVars: + SUPPORTED_ARCHITECTURES = dp.LocalVars["SUPPORTED_ARCHITECTURES"].split('|') + TARGET_ARCHITECTURES = environment.GetValue("TARGET_ARCH").split(' ') + + # Skip if there is no intersection between SUPPORTED_ARCHITECTURES and TARGET_ARCHITECTURES + if len(set(SUPPORTED_ARCHITECTURES) & set(TARGET_ARCHITECTURES)) == 0: + tc.SetSkipped() + tc.LogStdError("No supported architecutres to build for host unit tests") + return -1 + + uefiBuilder = UefiBuilder() + # do all the steps + # WorkSpace, PackagesPath, PInHelper, PInManager + ret = uefiBuilder.Go(Edk2pathObj.WorkspacePath, os.pathsep.join(Edk2pathObj.PackagePathList), PLMHelper, PLM) + if ret != 0: # failure: + tc.SetFailed("Compile failed for {0}".format(packagename), "Compile_FAILED") + tc.LogStdError("{0} Compile failed with error code {1} ".format(AP_Path, ret)) + return 1 + + else: + tc.SetSuccess() + return 0 diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/HostUnitTestCompilerPlugin/HostUnitTestCompiler_plug_in.yaml b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/HostUnitTestCompilerPlugin/HostUnitTestCompiler_plug_in.yaml new file mode 100644 index 00000000..dea9b056 --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/HostUnitTestCompilerPlugin/HostUnitTestCompiler_plug_in.yaml @@ -0,0 +1,12 @@ +## +# CiBuildPlugin used to build anything that identifies +# as a unit test. +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "host-based-test", + "name": "Host Unit Test Compiler Plugin", + "module": "HostUnitTestCompilerPlugin" +} diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/HostUnitTestCompilerPlugin/Readme.md b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/HostUnitTestCompilerPlugin/Readme.md new file mode 100644 index 00000000..f0def4c5 --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/HostUnitTestCompilerPlugin/Readme.md @@ -0,0 +1,24 @@ +# Host UnitTest Compiler Plugin + +A CiBuildPlugin that compiles the dsc for host based unit test apps. +An IUefiBuildPlugin may be attached to this plugin that will run the unit tests and collect the results after successful compilation. + +## Configuration + +The package relative path of the DSC file to build. + +``` yaml +"HostUnitTestCompilerPlugin": { + "DscPath": "<path to dsc from root of pkg>" +} +``` + +### DscPath + +Package relative path to the DSC file to build. + +## Copyright + +Copyright (c) Microsoft Corporation. +SPDX-License-Identifier: BSD-2-Clause-Patent + diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/HostUnitTestDscCompleteCheck/HostUnitTestDscCompleteCheck.py b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/HostUnitTestDscCompleteCheck/HostUnitTestDscCompleteCheck.py new file mode 100755 index 00000000..79dffe8f --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/HostUnitTestDscCompleteCheck/HostUnitTestDscCompleteCheck.py @@ -0,0 +1,140 @@ +# @file HostUnitTestDscCompleteCheck.py +# +# This is a copy of DscCompleteCheck with different filtering logic. +# It should be discussed if this should be one plugin +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +import logging +import os +from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin +from edk2toollib.uefi.edk2.parsers.dsc_parser import DscParser +from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser +from edk2toolext.environment.var_dict import VarDict + + +class HostUnitTestDscCompleteCheck(ICiBuildPlugin): + """ + A CiBuildPlugin that scans the package Host Unit Test dsc file and confirms all Host application modules (inf files) are + listed in the components sections. + + Configuration options: + "HostUnitTestDscCompleteCheck": { + "DscPath": "", # Path to Host based unit test DSC file + "IgnoreInf": [] # Ignore INF if found in filesystem but not dsc + } + """ + + def GetTestName(self, packagename: str, environment: VarDict) -> tuple: + """ Provide the testcase name and classname for use in reporting + + Args: + packagename: string containing name of package to build + environment: The VarDict for the test to run in + Returns: + a tuple containing the testcase name and the classname + (testcasename, classname) + testclassname: a descriptive string for the testcase can include whitespace + classname: should be patterned <packagename>.<plugin>.<optionally any unique condition> + """ + return ("Check the " + packagename + " Host Unit Test DSC for a being complete", packagename + ".HostUnitTestDscCompleteCheck") + + ## + # External function of plugin. This function is used to perform the task of the MuBuild Plugin + # + # - package is the edk2 path to package. This means workspace/packagepath relative. + # - edk2path object configured with workspace and packages path + # - PkgConfig Object (dict) for the pkg + # - VarDict containing the shell environment Build Vars + # - Plugin Manager Instance + # - Plugin Helper Obj Instance + # - Junit Logger + # - output_stream the StringIO output stream from this plugin via logging + def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None): + overall_status = 0 + + # Parse the config for required DscPath element + if "DscPath" not in pkgconfig: + tc.SetSkipped() + tc.LogStdError( + "DscPath not found in config file. Nothing to check.") + return -1 + + abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath( + packagename) + abs_dsc_path = os.path.join(abs_pkg_path, pkgconfig["DscPath"].strip()) + wsr_dsc_path = Edk2pathObj.GetEdk2RelativePathFromAbsolutePath( + abs_dsc_path) + + if abs_dsc_path is None or wsr_dsc_path == "" or not os.path.isfile(abs_dsc_path): + tc.SetSkipped() + tc.LogStdError("Package Host Unit Test Dsc not found") + return 0 + + # Get INF Files + INFFiles = self.WalkDirectoryForExtension([".inf"], abs_pkg_path) + INFFiles = [Edk2pathObj.GetEdk2RelativePathFromAbsolutePath( + x) for x in INFFiles] # make edk2relative path so can compare with DSC + + # remove ignores + + if "IgnoreInf" in pkgconfig: + for a in pkgconfig["IgnoreInf"]: + a = a.replace(os.sep, "/") + try: + tc.LogStdOut("Ignoring INF {0}".format(a)) + INFFiles.remove(a) + except: + tc.LogStdError( + "HostUnitTestDscCompleteCheck.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a)) + logging.info( + "HostUnitTestDscCompleteCheck.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a)) + + # DSC Parser + dp = DscParser() + dp.SetBaseAbsPath(Edk2pathObj.WorkspacePath) + dp.SetPackagePaths(Edk2pathObj.PackagePathList) + dp.SetInputVars(environment.GetAllBuildKeyValues()) + dp.ParseFile(wsr_dsc_path) + + # Check if INF in component section + for INF in INFFiles: + if not any(INF.strip() in x for x in dp.ThreeMods) and \ + not any(INF.strip() in x for x in dp.SixMods) and \ + not any(INF.strip() in x for x in dp.OtherMods): + + infp = InfParser().SetBaseAbsPath(Edk2pathObj.WorkspacePath) + infp.SetPackagePaths(Edk2pathObj.PackagePathList) + infp.ParseFile(INF) + if("MODULE_TYPE" not in infp.Dict): + tc.LogStdOut( + "Ignoring INF. Missing key for MODULE_TYPE {0}".format(INF)) + continue + + if(infp.Dict["MODULE_TYPE"] == "HOST_APPLICATION"): + # should compile test a library that is declared type HOST_APPLICATION + pass + + elif len(infp.SupportedPhases) > 0 and \ + "HOST_APPLICATION" in infp.SupportedPhases: + # should compile test a library that supports HOST_APPLICATION but + # require it to be an explicit opt-in + pass + + else: + tc.LogStdOut( + "Ignoring INF. MODULE_TYPE or suppored phases not HOST_APPLICATION {0}".format(INF)) + continue + + logging.critical(INF + " not in " + wsr_dsc_path) + tc.LogStdError("{0} not in {1}".format(INF, wsr_dsc_path)) + overall_status = overall_status + 1 + + # If XML object exists, add result + if overall_status != 0: + tc.SetFailed("HostUnitTestDscCompleteCheck {0} Failed. Errors {1}".format( + wsr_dsc_path, overall_status), "CHECK_FAILED") + else: + tc.SetSuccess() + return overall_status diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/HostUnitTestDscCompleteCheck/HostUnitTestDscCompleteCheck_plug_in.yaml b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/HostUnitTestDscCompleteCheck/HostUnitTestDscCompleteCheck_plug_in.yaml new file mode 100644 index 00000000..9e7dc2eb --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/HostUnitTestDscCompleteCheck/HostUnitTestDscCompleteCheck_plug_in.yaml @@ -0,0 +1,12 @@ +## +# CiBuildPlugin used to confirm all INFs are listed in +# the components section of package dsc +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "host-based-test", + "name": "Host Unit Test Dsc Complete Check Test", + "module": "HostUnitTestDscCompleteCheck" + } diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/HostUnitTestDscCompleteCheck/Readme.md b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/HostUnitTestDscCompleteCheck/Readme.md new file mode 100644 index 00000000..4a7fd0c9 --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/HostUnitTestDscCompleteCheck/Readme.md @@ -0,0 +1,32 @@ +# Host Unit Test Dsc Complete Check Plugin + +This CiBuildPlugin scans all INF files from a package for those related to host +based unit tests confirms they are listed in the unit test DSC file for the package. +The test considers it an error if any INF meeting the requirements does not appear +in the `Components` section of the unit test DSC. This is critical because +much of the CI infrastructure assumes that modules will be listed in the DSC +and compiled. + +This test will only require INFs in the following cases: + +1. When MODULE_TYPE = HOST_APPLICATION +2. When a Library instance supports the HOST_APPLICATION environment + +## Configuration + +The plugin has a few configuration options to support the UEFI codebase. + +``` yaml +"HostUnitTestDscCompleteCheck": { + "DscPath": "", # Path to Host based unit test DSC file + "IgnoreInf": [] # Ignore INF if found in filesystem but not dsc +} +``` + +### DscPath + +Path to DSC to consider platform dsc + +### IgnoreInf + +Ignore error if Inf file is not listed in DSC file diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/LibraryClassCheck/LibraryClassCheck.py b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/LibraryClassCheck/LibraryClassCheck.py new file mode 100755 index 00000000..57020856 --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/LibraryClassCheck/LibraryClassCheck.py @@ -0,0 +1,153 @@ +# @file LibraryClassCheck.py +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +import logging +import os +from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin +from edk2toollib.uefi.edk2.parsers.dec_parser import DecParser +from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser +from edk2toolext.environment.var_dict import VarDict + + +class LibraryClassCheck(ICiBuildPlugin): + """ + A CiBuildPlugin that scans the code tree and library classes for undeclared + files + + Configuration options: + "LibraryClassCheck": { + IgnoreHeaderFile: [], # Ignore a file found on disk + IgnoreLibraryClass: [] # Ignore a declaration found in dec file + } + """ + + def GetTestName(self, packagename: str, environment: VarDict) -> tuple: + """ Provide the testcase name and classname for use in reporting + testclassname: a descriptive string for the testcase can include whitespace + classname: should be patterned <packagename>.<plugin>.<optionally any unique condition> + + Args: + packagename: string containing name of package to build + environment: The VarDict for the test to run in + Returns: + a tuple containing the testcase name and the classname + (testcasename, classname) + """ + return ("Check library class declarations in " + packagename, packagename + ".LibraryClassCheck") + + def __GetPkgDec(self, rootpath): + try: + allEntries = os.listdir(rootpath) + for entry in allEntries: + if entry.lower().endswith(".dec"): + return(os.path.join(rootpath, entry)) + except Exception: + logging.error("Unable to find DEC for package:{0}".format(rootpath)) + + return None + + ## + # External function of plugin. This function is used to perform the task of the MuBuild Plugin + # + # - package is the edk2 path to package. This means workspace/packagepath relative. + # - edk2path object configured with workspace and packages path + # - PkgConfig Object (dict) for the pkg + # - EnvConfig Object + # - Plugin Manager Instance + # - Plugin Helper Obj Instance + # - Junit Logger + # - output_stream the StringIO output stream from this plugin via logging + def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None): + overall_status = 0 + LibraryClassIgnore = [] + + abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagename) + abs_dec_path = self.__GetPkgDec(abs_pkg_path) + wsr_dec_path = Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(abs_dec_path) + + if abs_dec_path is None or wsr_dec_path == "" or not os.path.isfile(abs_dec_path): + tc.SetSkipped() + tc.LogStdError("No DEC file {0} in package {1}".format(abs_dec_path, abs_pkg_path)) + return -1 + + # Get all include folders + dec = DecParser() + dec.SetBaseAbsPath(Edk2pathObj.WorkspacePath).SetPackagePaths(Edk2pathObj.PackagePathList) + dec.ParseFile(wsr_dec_path) + + AllHeaderFiles = [] + + for includepath in dec.IncludePaths: + ## Get all header files in the library folder + AbsLibraryIncludePath = os.path.join(abs_pkg_path, includepath, "Library") + if(not os.path.isdir(AbsLibraryIncludePath)): + continue + + hfiles = self.WalkDirectoryForExtension([".h"], AbsLibraryIncludePath) + hfiles = [os.path.relpath(x,abs_pkg_path) for x in hfiles] # make package root relative path + hfiles = [x.replace("\\", "/") for x in hfiles] # make package relative path + + AllHeaderFiles.extend(hfiles) + + if len(AllHeaderFiles) == 0: + tc.SetSkipped() + tc.LogStdError(f"No Library include folder in any Include path") + return -1 + + # Remove ignored paths + if "IgnoreHeaderFile" in pkgconfig: + for a in pkgconfig["IgnoreHeaderFile"]: + try: + tc.LogStdOut("Ignoring Library Header File {0}".format(a)) + AllHeaderFiles.remove(a) + except: + tc.LogStdError("LibraryClassCheck.IgnoreHeaderFile -> {0} not found. Invalid Header File".format(a)) + logging.info("LibraryClassCheck.IgnoreHeaderFile -> {0} not found. Invalid Header File".format(a)) + + if "IgnoreLibraryClass" in pkgconfig: + LibraryClassIgnore = pkgconfig["IgnoreLibraryClass"] + + + ## Attempt to find library classes + for lcd in dec.LibraryClasses: + ## Check for correct file path separator + if "\\" in lcd.path: + tc.LogStdError("LibraryClassCheck.DecFilePathSeparator -> {0} invalid.".format(lcd.path)) + logging.error("LibraryClassCheck.DecFilePathSeparator -> {0} invalid.".format(lcd.path)) + overall_status += 1 + continue + + if lcd.name in LibraryClassIgnore: + tc.LogStdOut("Ignoring Library Class Name {0}".format(lcd.name)) + LibraryClassIgnore.remove(lcd.name) + continue + + logging.debug(f"Looking for Library Class {lcd.path}") + try: + AllHeaderFiles.remove(lcd.path) + + except ValueError: + tc.LogStdError(f"Library {lcd.name} with path {lcd.path} not found in package filesystem") + logging.error(f"Library {lcd.name} with path {lcd.path} not found in package filesystem") + overall_status += 1 + + ## any remaining AllHeaderFiles are not described in DEC + for h in AllHeaderFiles: + tc.LogStdError(f"Library Header File {h} not declared in package DEC {wsr_dec_path}") + logging.error(f"Library Header File {h} not declared in package DEC {wsr_dec_path}") + overall_status += 1 + + ## Warn about any invalid library class names in the ignore list + for r in LibraryClassIgnore: + tc.LogStdError("LibraryClassCheck.IgnoreLibraryClass -> {0} not found. Library Class not found".format(r)) + logging.info("LibraryClassCheck.IgnoreLibraryClass -> {0} not found. Library Class not found".format(r)) + + + # If XML object exists, add result + if overall_status != 0: + tc.SetFailed("LibraryClassCheck {0} Failed. Errors {1}".format(wsr_dec_path, overall_status), "CHECK_FAILED") + else: + tc.SetSuccess() + return overall_status diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/LibraryClassCheck/LibraryClassCheck_plug_in.yaml b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/LibraryClassCheck/LibraryClassCheck_plug_in.yaml new file mode 100644 index 00000000..db35f22e --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/LibraryClassCheck/LibraryClassCheck_plug_in.yaml @@ -0,0 +1,11 @@ +## @file +# CiBuildPlugin used to check that all library classes are declared correctly in dec file +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "cibuild", + "name": "Library Class Check Test", + "module": "LibraryClassCheck" +} diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/LibraryClassCheck/Readme.md b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/LibraryClassCheck/Readme.md new file mode 100644 index 00000000..95f22e7b --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/LibraryClassCheck/Readme.md @@ -0,0 +1,25 @@ +# Library Class Check Plugin + +This CiBuildPlugin scans at all library header files found in the `Library` +folders in all of the package's declared include directories and ensures that +all files have a matching LibraryClass declaration in the DEC file for the +package. Any missing declarations will cause a failure. + +## Configuration + +The plugin has a few configuration options to support the UEFI codebase. + +``` yaml +"LibraryClassCheck": { + IgnoreHeaderFile: [], # Ignore a file found on disk + IgnoreLibraryClass: [] # Ignore a declaration found in dec file +} +``` + +### IgnoreHeaderFile + +Ignore a file found on disk + +### IgnoreLibraryClass + +Ignore a declaration found in dec file diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/LicenseCheck/LicenseCheck.py b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/LicenseCheck/LicenseCheck.py new file mode 100755 index 00000000..e19e164e --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/LicenseCheck/LicenseCheck.py @@ -0,0 +1,115 @@ +# @file LicenseCheck.py +# +# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR> +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +import os +import logging +import re +from io import StringIO +from typing import List, Tuple +from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin +from edk2toolext.environment.var_dict import VarDict +from edk2toollib.utility_functions import RunCmd + + +class LicenseCheck(ICiBuildPlugin): + + """ + A CiBuildPlugin to check the license for new added files. + + Configuration options: + "LicenseCheck": { + "IgnoreFiles": [] + }, + """ + + license_format_preflix = 'SPDX-License-Identifier' + + bsd2_patent = 'BSD-2-Clause-Patent' + + Readdedfileformat = re.compile(r'\+\+\+ b\/(.*)') + + file_extension_list = [".c", ".h", ".inf", ".dsc", ".dec", ".py", ".bat", ".sh", ".uni", ".yaml", + ".fdf", ".inc", "yml", ".asm", ".asm16", ".asl", ".vfr", ".s", ".S", ".aslc", + ".nasm", ".nasmb", ".idf", ".Vfr", ".H"] + + def GetTestName(self, packagename: str, environment: VarDict) -> tuple: + """ Provide the testcase name and classname for use in reporting + testclassname: a descriptive string for the testcase can include whitespace + classname: should be patterned <packagename>.<plugin>.<optionally any unique condition> + + Args: + packagename: string containing name of package to build + environment: The VarDict for the test to run in + Returns: + a tuple containing the testcase name and the classname + (testcasename, classname) + """ + return ("Check for license for " + packagename, packagename + ".LicenseCheck") + + ## + # External function of plugin. This function is used to perform the task of the ci_build_plugin Plugin + # + # - package is the edk2 path to package. This means workspace/packagepath relative. + # - edk2path object configured with workspace and packages path + # - PkgConfig Object (dict) for the pkg + # - EnvConfig Object + # - Plugin Manager Instance + # - Plugin Helper Obj Instance + # - Junit Logger + # - output_stream the StringIO output stream from this plugin via logging + def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None): + return_buffer = StringIO() + params = "diff --unified=0 origin/master HEAD" + RunCmd("git", params, outstream=return_buffer) + p = return_buffer.getvalue().strip() + patch = p.split("\n") + return_buffer.close() + + ignore_files = [] + if "IgnoreFiles" in pkgconfig: + ignore_files = pkgconfig["IgnoreFiles"] + + self.ok = True + self.startcheck = False + self.license = True + self.all_file_pass = True + count = len(patch) + line_index = 0 + for line in patch: + if line.startswith('--- /dev/null'): + nextline = patch[line_index + 1] + added_file = self.Readdedfileformat.search(nextline).group(1) + added_file_extension = os.path.splitext(added_file)[1] + if added_file_extension in self.file_extension_list and packagename in added_file: + if (self.IsIgnoreFile(added_file, ignore_files)): + line_index = line_index + 1 + continue + self.startcheck = True + self.license = False + if self.startcheck and self.license_format_preflix in line: + if self.bsd2_patent in line: + self.license = True + if line_index + 1 == count or patch[line_index + 1].startswith('diff --') and self.startcheck: + if not self.license: + self.all_file_pass = False + error_message = "Invalid license in: " + added_file + " Hint: Only BSD-2-Clause-Patent is accepted." + logging.error(error_message) + self.startcheck = False + self.license = True + line_index = line_index + 1 + + if self.all_file_pass: + tc.SetSuccess() + return 0 + else: + tc.SetFailed("License Check {0} Failed. ".format(packagename), "LICENSE_CHECK_FAILED") + return 1 + + def IsIgnoreFile(self, file: str, ignore_files: List[str]) -> bool: + for f in ignore_files: + if f in file: + return True + return False diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/LicenseCheck/LicenseCheck_plug_in.yaml b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/LicenseCheck/LicenseCheck_plug_in.yaml new file mode 100644 index 00000000..6bb8225b --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/LicenseCheck/LicenseCheck_plug_in.yaml @@ -0,0 +1,11 @@ +## @file +# CiBuildPlugin used to check license issues for new added files +# +# Copyright (c) 2020, Intel Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "cibuild", + "name": "License Check Test", + "module": "LicenseCheck" +} diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/LicenseCheck/Readme.md b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/LicenseCheck/Readme.md new file mode 100644 index 00000000..ddb7670a --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/LicenseCheck/Readme.md @@ -0,0 +1,17 @@ +# License Check Plugin + +This CiBuildPlugin scans all new added files in a package to make sure code +is contributed under BSD-2-Clause-Patent. + +## Configuration + +The plugin can be configured to ignore certain files. + +``` yaml +"LicenseCheck": { + "IgnoreFiles": [] +} +``` +### IgnoreFiles + +OPTIONAL List of file to ignore. diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/SpellCheck/Readme.md b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/SpellCheck/Readme.md new file mode 100644 index 00000000..688c3e88 --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/SpellCheck/Readme.md @@ -0,0 +1,127 @@ +# Spell Check Plugin + +This CiBuildPlugin scans all the files in a given package and checks for +spelling errors. + +This plugin requires NodeJs and cspell. If the plugin doesn't find its required +tools then it will mark the test as skipped. + +* NodeJS: https://nodejs.org/en/ +* cspell: https://www.npmjs.com/package/cspell + * Src and doc available: https://github.com/streetsidesoftware/cspell + +## Configuration + +The plugin has a few configuration options to support the UEFI codebase. + +``` yaml + "SpellCheck": { + "AuditOnly": False, # If True, log all errors and then mark as skipped + "IgnoreFiles": [], # use gitignore syntax to ignore errors in matching files + "ExtendWords": [], # words to extend to the dictionary for this package + "IgnoreStandardPaths": [], # Standard Plugin defined paths that should be ignore + "AdditionalIncludePaths": [] # Additional paths to spell check (wildcards supported) + } +``` + +### AuditOnly + +Boolean - Default is False. +If True run the test in an Audit only mode which will log all errors but instead +of failing the build it will set the test as skipped. This allows visibility +into the failures without breaking the build. + +### IgnoreFiles + +This supports .gitignore file and folder matching strings including wildcards + +* All files will be parsed regardless but then any spelling errors found within + ignored files will not be reported as an error. +* Errors in ignored files will still be output to the test results as + informational comments. + +### ExtendWords + +This list allows words to be added to the dictionary for the spell checker when +this package is tested. These follow the rules of the cspell config words field. + +### IgnoreStandardPaths + +This plugin by default will check the below standard paths. If the package +would like to ignore any of them list that here. + +```python +[ +# C source +"*.c", +"*.h", + +# Assembly files +"*.nasm", +"*.asm", +"*.masm", +"*.s", + +# ACPI source language +"*.asl", + +# Edk2 build files +"*.dsc", "*.dec", "*.fdf", "*.inf", + +# Documentation files +"*.md", "*.txt" +] +``` + +### AdditionalIncludePaths + +If the package would to add additional path patterns to be included in +spellchecking they can be defined here. + +## Other configuration + +In the cspell.base.json there are numerous other settings configured. There is +no support to override these on a per package basis but future features could +make this available. One interesting configuration option is `minWordLength`. +Currently it is set to _5_ which means all 2,3, and 4 letter words will be +ignored. This helps minimize the number of technical acronyms, register names, +and other UEFI specific values that must be ignored. + +## False positives + +The cspell dictionary is not perfect and there are cases where technical words +or acronyms are not found in the dictionary. There are three ways to resolve +false positives and the choice for which method should be based on how broadly +the word should be accepted. + +### CSpell Base Config file + +If the change should apply to all UEFI code and documentation then it should be +added to the base config file `words` section. The base config file is adjacent +to this file and titled `cspell.base.json`. This is a list of accepted words +for all spell checking operations on all packages. + +### Package Config + +In the package `*.ci.yaml` file there is a `SpellCheck` config section. This +section allows files to be ignored as well as words that should be considered +valid for all files within this package. Add the desired words to the +"ExtendedWords" member. + +### In-line File + +CSpell supports numerous methods to annotate your files to ignore words, +sections, etc. This can be found in CSpell documentation. Suggestion here is +to use a c-style comment at the top of the file to add words that should be +ignored just for this file. Obviously this has the highest maintenance cost so +it should only be used for file unique words. + +``` c +// spell-checker:ignore unenroll, word2, word3 +``` + +or + +```ini +# spell-checker:ignore unenroll, word2, word3 +``` diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/SpellCheck/SpellCheck.py b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/SpellCheck/SpellCheck.py new file mode 100755 index 00000000..3a7fdd2f --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/SpellCheck/SpellCheck.py @@ -0,0 +1,216 @@ +# @file SpellCheck.py +# +# An edk2-pytool based plugin wrapper for cspell +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +import logging +import json +import yaml +from io import StringIO +import os +from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin +from edk2toollib.utility_functions import RunCmd +from edk2toolext.environment.var_dict import VarDict +from edk2toollib.gitignore_parser import parse_gitignore_lines +from edk2toolext.environment import version_aggregator + + +class SpellCheck(ICiBuildPlugin): + """ + A CiBuildPlugin that uses the cspell node module to scan the files + from the package being tested for spelling errors. The plugin contains + the base cspell.json file then thru the configuration options other settings + can be changed or extended. + + Configuration options: + "SpellCheck": { + "AuditOnly": False, # Don't fail the build if there are errors. Just log them + "IgnoreFiles": [], # use gitignore syntax to ignore errors in matching files + "ExtendWords": [], # words to extend to the dictionary for this package + "IgnoreStandardPaths": [], # Standard Plugin defined paths that should be ignore + "AdditionalIncludePaths": [] # Additional paths to spell check (wildcards supported) + } + """ + + # + # A package can remove any of these using IgnoreStandardPaths + # + STANDARD_PLUGIN_DEFINED_PATHS = ["*.c", "*.h", + "*.nasm", "*.asm", "*.masm", "*.s", + "*.asl", + "*.dsc", "*.dec", "*.fdf", "*.inf", + "*.md", "*.txt" + ] + + def GetTestName(self, packagename: str, environment: VarDict) -> tuple: + """ Provide the testcase name and classname for use in reporting + + Args: + packagename: string containing name of package to build + environment: The VarDict for the test to run in + Returns: + a tuple containing the testcase name and the classname + (testcasename, classname) + testclassname: a descriptive string for the testcase can include whitespace + classname: should be patterned <packagename>.<plugin>.<optionally any unique condition> + """ + return ("Spell check files in " + packagename, packagename + ".SpellCheck") + + ## + # External function of plugin. This function is used to perform the task of the CiBuild Plugin + # + # - package is the edk2 path to package. This means workspace/packagepath relative. + # - edk2path object configured with workspace and packages path + # - PkgConfig Object (dict) for the pkg + # - EnvConfig Object + # - Plugin Manager Instance + # - Plugin Helper Obj Instance + # - Junit Logger + # - output_stream the StringIO output stream from this plugin via logging + + def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None): + Errors = [] + + abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath( + packagename) + + if abs_pkg_path is None: + tc.SetSkipped() + tc.LogStdError("No package {0}".format(packagename)) + return -1 + + # check for node + return_buffer = StringIO() + ret = RunCmd("node", "--version", outstream=return_buffer) + if (ret != 0): + tc.SetSkipped() + tc.LogStdError("NodeJs not installed. Test can't run") + logging.warning("NodeJs not installed. Test can't run") + return -1 + node_version = return_buffer.getvalue().strip() # format vXX.XX.XX + tc.LogStdOut(f"Node version: {node_version}") + version_aggregator.GetVersionAggregator().ReportVersion( + "NodeJs", node_version, version_aggregator.VersionTypes.INFO) + + # Check for cspell + return_buffer = StringIO() + ret = RunCmd("cspell", "--version", outstream=return_buffer) + if (ret != 0): + tc.SetSkipped() + tc.LogStdError("cspell not installed. Test can't run") + logging.warning("cspell not installed. Test can't run") + return -1 + cspell_version = return_buffer.getvalue().strip() # format XX.XX.XX + tc.LogStdOut(f"CSpell version: {cspell_version}") + version_aggregator.GetVersionAggregator().ReportVersion( + "CSpell", cspell_version, version_aggregator.VersionTypes.INFO) + + package_relative_paths_to_spell_check = SpellCheck.STANDARD_PLUGIN_DEFINED_PATHS + + # + # Allow the ci.yaml to remove any of the above standard paths + # + if("IgnoreStandardPaths" in pkgconfig): + for a in pkgconfig["IgnoreStandardPaths"]: + if(a in package_relative_paths_to_spell_check): + tc.LogStdOut( + f"ignoring standard path due to ci.yaml ignore: {a}") + package_relative_paths_to_spell_check.remove(a) + else: + tc.LogStdOut(f"Invalid IgnoreStandardPaths value: {a}") + + # + # check for any additional include paths defined by package config + # + if("AdditionalIncludePaths" in pkgconfig): + package_relative_paths_to_spell_check.extend( + pkgconfig["AdditionalIncludePaths"]) + + # + # Make the path string for cspell to check + # + relpath = os.path.relpath(abs_pkg_path) + cpsell_paths = " ".join( + [f"{relpath}/**/{x}" for x in package_relative_paths_to_spell_check]) + + # Make the config file + config_file_path = os.path.join( + Edk2pathObj.WorkspacePath, "Build", packagename, "cspell_actual_config.json") + mydir = os.path.dirname(os.path.abspath(__file__)) + # load as yaml so it can have comments + base = os.path.join(mydir, "cspell.base.yaml") + with open(base, "r") as i: + config = yaml.safe_load(i) + + if("ExtendWords" in pkgconfig): + config["words"].extend(pkgconfig["ExtendWords"]) + with open(config_file_path, "w") as o: + json.dump(config, o) # output as json so compat with cspell + + All_Ignores = [] + # Parse the config for other ignores + if "IgnoreFiles" in pkgconfig: + All_Ignores.extend(pkgconfig["IgnoreFiles"]) + + # spell check all the files + ignore = parse_gitignore_lines(All_Ignores, os.path.join( + abs_pkg_path, "nofile.txt"), abs_pkg_path) + + # result is a list of strings like this + # C:\src\sp-edk2\edk2\FmpDevicePkg\FmpDevicePkg.dec:53:9 - Unknown word (Capule) + EasyFix = [] + results = self._check_spelling(cpsell_paths, config_file_path) + for r in results: + path, _, word = r.partition(" - Unknown word ") + if len(word) == 0: + # didn't find pattern + continue + + pathinfo = path.rsplit(":", 2) # remove the line no info + if(ignore(pathinfo[0])): # check against ignore list + tc.LogStdOut(f"ignoring error due to ci.yaml ignore: {r}") + continue + + # real error + EasyFix.append(word.strip().strip("()")) + Errors.append(r) + + # Log all errors tc StdError + for l in Errors: + tc.LogStdError(l.strip()) + + # Helper - Log the syntax needed to add these words to dictionary + if len(EasyFix) > 0: + EasyFix = sorted(set(a.lower() for a in EasyFix)) + tc.LogStdOut("\n Easy fix:") + OneString = "If these are not errors add this to your ci.yaml file.\n" + OneString += '"SpellCheck": {\n "ExtendWords": [' + for a in EasyFix: + tc.LogStdOut(f'\n"{a}",') + OneString += f'\n "{a}",' + logging.info(OneString.rstrip(",") + '\n ]\n}') + + # add result to test case + overall_status = len(Errors) + if overall_status != 0: + if "AuditOnly" in pkgconfig and pkgconfig["AuditOnly"]: + # set as skipped if AuditOnly + tc.SetSkipped() + return -1 + else: + tc.SetFailed("SpellCheck {0} Failed. Errors {1}".format( + packagename, overall_status), "CHECK_FAILED") + else: + tc.SetSuccess() + return overall_status + + def _check_spelling(self, abs_file_to_check: str, abs_config_file_to_use: str) -> []: + output = StringIO() + ret = RunCmd( + "cspell", f"--config {abs_config_file_to_use} {abs_file_to_check}", outstream=output) + if ret == 0: + return [] + else: + return output.getvalue().strip().splitlines() diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/SpellCheck/SpellCheck_plug_in.yaml b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/SpellCheck/SpellCheck_plug_in.yaml new file mode 100644 index 00000000..b2c1cc52 --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/SpellCheck/SpellCheck_plug_in.yaml @@ -0,0 +1,11 @@ +## @file +# CiBuildPlugin used to check spelling +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "scope": "cibuild", + "name": "Spell Check Test", + "module": "SpellCheck" +} diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/SpellCheck/cspell.base.yaml b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/SpellCheck/cspell.base.yaml new file mode 100644 index 00000000..237a59c9 --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Plugin/SpellCheck/cspell.base.yaml @@ -0,0 +1,183 @@ +## @file +# CSpell configuration +# +# Copyright (c) Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +{ + "version": "0.1", + "language": "en", + "dictionaries": [ + "companies ", + "softwareTerms", + "python", + "cpp" + ], + "ignorePaths": [ + "*.pdb", + "**/*_extdep/**", + "*.pdf", + "*.exe", + "*.jpg" + ], + "minWordLength": 5, + "allowCompoundWords": false, + "maxNumberOfProblems": 200, + "maxDuplicateProblems": 200, + "ignoreWords": [ + "muchange" + ], + "words": [ + "MTRRs", + "Microarchitecture", + "Goldmont", + "cpuid", + "mwait", + "cstate", + "smram", + "scrtm", + "smbus", + "selftest", + "socket", + "MMRAM", + "qword", + "ENDBR", + "SMBASE", + "FXSAVE", + "FXRSTOR", + "RDRAND", + "IOAPIC", + "ATAPI", + "movsb", + "iretw", + "XENSTORE", + "cdrom", + "oprom", + "oproms", + "varstore", + "EKU", + "ascii", + "nmake", + "NVDIMM", + "nasmb", + "Mtftp", + "Hypercall", + "hypercalls", + "IOMMU", + "QEMU", + "qemus", + "OVMF", + "tiano", + "tianocore", + "edkii", + "coreboot", + "uefipayload", + "bootloader", + "bootloaders", + "mdepkg", + "skuid", + "dxefv", + "toolchain", + "libraryclass", + "preboot", + "pythonpath", + "cygpath", + "nuget", + "basetools", + "prepi", + "OPTEE", + "stringid", + "peims", + "memmap", + "guids", + "uuids", + "smbios", + "certdb", + "certdbv", + "EfiSigList", + "depex", + "IHANDLE", + "Virtio", + "Mbytes", + "Citrix", + "initrd", + "semihost", + "Semihosting", + "Trustzone", + "Fastboot", + "framebuffer", + "genfw", + "TTYTERM", + "miniport", + "LFENCE", + "PCANSI", + "submodule", + "submodules", + "brotli", + "PCCTS", + "softfloat", + "whitepaper", + "ACPICA", + "plugfest", + "bringup", + "formset", #VFR + "ideqvallist", + "numberof", + "oneof", + "endformset", + "endnumeric", + "endoneof", + "disableif", + "guidid", + "classguid", + "efivarstore", + "formsetguid", + "formid", + "suppressif", + "grayoutif", + "ideqval", + "endform", + "endcheckbox", + "questionid", + "questionref", + "enddate", + "endstring", + "guidop", + "endguidop", + "langdef", + "dynamicex", + "tokenspace", + "tokenguid", + "pcd's", #seems like cspell bug + "peim's", + "autogen", + "Disasm", + "Torito", + "SRIOV", + "MRIOV", + "UARTs", + "Consplitter", # common module in UEFI + "FIFOs", + "ACPINVS", + "Endof", # due to of not being uppercase + "bootability", + "Sdhci", + "inmodule", + "RISCV", + "edksetup", + "iscsi", + "nvdata", + "pytools", + "NTDDI", + "Wnonportable", + "CLANGPDB", + "nologo", + "lldmap", + "ASMLINK", + "NODEFAULTLIB", + "vcruntimed", + "ucrtd", + "msvcrtd", + "XIPFLAGS" + ] +} diff --git a/src/VBox/Devices/EFI/Firmware/.pytool/Readme.md b/src/VBox/Devices/EFI/Firmware/.pytool/Readme.md new file mode 100644 index 00000000..d79d233a --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/.pytool/Readme.md @@ -0,0 +1,294 @@ +# Edk2 Continuous Integration + +## Basic Status + +| Package | Windows VS2019 (IA32/X64)| Ubuntu GCC (IA32/X64/ARM/AARCH64) | Known Issues | +| :---- | :----- | :---- | :--- | +| ArmPkg | | :heavy_check_mark: | +| ArmPlatformPkg | | :heavy_check_mark: | +| ArmVirtPkg | SEE PACKAGE README | SEE PACKAGE README | +| CryptoPkg | :heavy_check_mark: | :heavy_check_mark: | Spell checking in audit mode +| DynamicTablesPkg | | :heavy_check_mark: | +| EmbeddedPkg | +| EmulatorPkg | SEE PACKAGE README | SEE PACKAGE README | Spell checking in audit mode +| FatPkg | :heavy_check_mark: | :heavy_check_mark: | +| FmpDevicePkg | :heavy_check_mark: | :heavy_check_mark: | +| IntelFsp2Pkg | +| IntelFsp2WrapperPkg | +| MdeModulePkg | :heavy_check_mark: | :heavy_check_mark: | DxeIpl dependency on ArmPkg, Depends on StandaloneMmPkg, Spell checking in audit mode +| MdePkg | :heavy_check_mark: | :heavy_check_mark: | Spell checking in audit mode +| NetworkPkg | :heavy_check_mark: | :heavy_check_mark: | Spell checking in audit mode +| OvmfPkg | SEE PACKAGE README | SEE PACKAGE README | Spell checking in audit mode +| PcAtChipsetPkg | :heavy_check_mark: | :heavy_check_mark: | +| SecurityPkg | :heavy_check_mark: | :heavy_check_mark: | Spell checking in audit mode +| ShellPkg | :heavy_check_mark: | :heavy_check_mark: | Spell checking in audit mode, 3 modules are not being built by DSC +| SignedCapsulePkg | +| SourceLevelDebugPkg | +| StandaloneMmPkg | :heavy_check_mark: | :heavy_check_mark: | +| UefiCpuPkg | :heavy_check_mark: | :heavy_check_mark: | Spell checking in audit mode, 2 binary modules not being built by DSC +| UefiPayloadPkg | +| UnitTestFrameworkPkg | :heavy_check_mark: | :heavy_check_mark: | + +For more detailed status look at the test results of the latest CI run on the +repo readme. + +## Background + +This Continuous integration and testing infrastructure leverages the TianoCore EDKII Tools PIP modules: +[library](https://pypi.org/project/edk2-pytool-library/) and +[extensions](https://pypi.org/project/edk2-pytool-extensions/) (with repos +located [here](https://github.com/tianocore/edk2-pytool-library) and +[here](https://github.com/tianocore/edk2-pytool-extensions)). + +The primary execution flows can be found in the +`.azurepipelines/Windows-VS2019.yml` and `.azurepipelines/Ubuntu-GCC5.yml` +files. These YAML files are consumed by the Azure Dev Ops Build Pipeline and +dictate what server resources should be used, how they should be configured, and +what processes should be run on them. An overview of this schema can be found +[here](https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=azure-devops&tabs=schema). + +Inspection of these files reveals the EDKII Tools commands that make up the +primary processes for the CI build: 'stuart_setup', 'stuart_update', and +'stuart_ci_build'. These commands come from the EDKII Tools PIP modules and are +configured as described below. More documentation on the tools can be +found [here](https://github.com/tianocore/edk2-pytool-extensions/blob/master/docs/using.md) +and [here](https://github.com/tianocore/edk2-pytool-extensions/blob/master/docs/features/feature_invocables.md). + +## Configuration + +Configuration of the CI process consists of (in order of precedence): + +* command-line arguments passed in via the Pipeline YAML +* a per-package configuration file (e.g. `<package-name>.ci.yaml`) that is + detected by the CI system in EDKII Tools. +* a global configuration Python module (e.g. `CISetting.py`) passed in via the + command-line + +The global configuration file is described in +[this readme](https://github.com/tianocore/edk2-pytool-extensions/blob/master/docs/usability/using_settings_manager.md) +from the EDKII Tools documentation. This configuration is written as a Python +module so that decisions can be made dynamically based on command line +parameters and codebase state. + +The per-package configuration file can override most settings in the global +configuration file, but is not dynamic. This file can be used to skip or +customize tests that may be incompatible with a specific package. Each test generally requires +per package configuration which comes from this file. + +## Running CI locally + +The EDKII Tools environment (and by extension the ci) is designed to support +easily and consistently running locally and in a cloud ci environment. To do +that a few steps should be followed. Details of EDKII Tools can be found in the +[docs folder here](https://github.com/tianocore/edk2-pytool-extensions/tree/master/docs) + +### Prerequisets + +1. A supported toolchain (others might work but this is what is tested and validated) + * Windows 10: + * VS 2017 or VS 2019 + * Windows SDK (for rc) + * Windows WDK (for capsules) + * Ubuntu 18.04 or Fedora + * GCC5 + * Easy to add more but this is the current state +2. Python 3.7.x or newer on path +3. git on path +4. Recommended to setup and activate a python virtual environment +5. Install the requirements `pip install --upgrade pip-requirements.txt` + +### Running CI + +1. clone your edk2 repo +2. Activate your python virtual environment in cmd window +3. Get code dependencies (done only when submodules change) + * `stuart_setup -c .pytool/CISettings.py TOOL_CHAIN_TAG=<your tag here>` +4. Update other dependencies (done more often) + * `stuart_update -c .pytool/CISettings.py TOOL_CHAIN_TAG=<your tag here>` +5. Run CI build (--help will give you options) + * `stuart_ci_build -c .pytool/CISettings.py TOOL_CHAIN_TAG=<your tag here>` + * -p <pkg1,pkg2,pkg3> : To build only certain packages use a CSV list + * -a <arch1,arch2,arch3>: To run only certain architectures use a CSV list + * -t <target1,target2>: To run only tests related to certain targets use a + CSV list + * By default all tests are opted in. Then given a package.ci.yaml file those + tests can be configured for a package. Finally setting the check to the + value `skip` will skip that plugin. Examples: + * `CompilerPlugin=skip` skip the build test + * `GuidCheck=skip` skip the Guid check + * `SpellCheck=skip` skip the spell checker + * etc +6. Detailed reports and logs per package are captured in the `Build` directory + +## Current PyTool Test Capabilities + +All CI tests are instances of EDKII Tools plugins. Documentation on the plugin +system can be found [here](https://github.com/tianocore/edk2-pytool-extensions/blob/master/docs/usability/using_plugin_manager.md) +and [here](https://github.com/tianocore/edk2-pytool-extensions/blob/master/docs/features/feature_plugin_manager.md). +Upon invocation, each plugin will be passed the path to the current package +under test and a dictionary containing its targeted configuration, as assembled +from the command line, per-package configuration, and global configuration. + +Note: CI plugins are considered unique from build plugins and helper plugins, +even though some CI plugins may execute steps of a build. + +In the example, these plugins live alongside the code under test (in the +`.pytool/Plugin` directory), but may be moved to the 'edk2-test' repo if that +location makes more sense for the community. + +### Module Inclusion Test - DscCompleteCheck + +This scans all INF files from a package and confirms they are +listed in the package level DSC file. The test considers it an error if any INF +does not appear in the `Components` section of the package-level DSC (indicating +that it would not be built if the package were built). This is critical because +much of the CI infrastructure assumes that all modules will be listed in the DSC +and compiled. + +This test will ignore INFs in the following cases: + +1. When `MODULE_TYPE` = `HOST_APPLICATION` +2. When a Library instance **only** supports the `HOST_APPLICATION` environment + +### Host Module Inclusion Test - HostUnitTestDscCompleteCheck + +This test scans all INF files from a package for those related to host +based unit tests and confirms they are listed in the unit test DSC file for the package. +The test considers it an error if any INF meeting the requirements does not appear +in the `Components` section of the unit test DSC. This is critical because +much of the CI infrastructure assumes that modules will be listed in the DSC +and compiled. + +This test will only require INFs in the following cases: + +1. When `MODULE_TYPE` = `HOST_APPLICATION` +2. When a Library instance explicitly supports the `HOST_APPLICATION` environment + +### Code Compilation Test - CompilerPlugin + +Once the Module Inclusion Test has verified that all modules would be built if +all package-level DSCs were built, the Code Compilation Test simply runs through +and builds every package-level DSC on every toolchain and for every architecture +that is supported. Any module that fails to build is considered an error. + +### Host Unit Test Compilation and Run Test - HostUnitTestCompilerPlugin + +A test that compiles the dsc for host based unit test apps. +On Windows this will also enable a build plugin to execute that will run the unit tests and verify the results. + +These tools will be invoked on any CI +pass that includes the NOOPT target. In order for these tools to do their job, +the package and tests must be configured in a particular way... + +#### Including Host-Based Tests in the Package YAML + +For example, looking at the `MdeModulePkg.ci.yaml` config file, there are two +config options that control HostBased test behavior: + +```json + ## options defined .pytool/Plugin/HostUnitTestCompilerPlugin + "HostUnitTestCompilerPlugin": { + "DscPath": "Test/MdeModulePkgHostTest.dsc" + }, +``` + +This option tell the test builder to run. The test builder needs to know which +modules in this package are host-based tests, so that DSC path is provided. + +#### Configuring the HostBased DSC + +The HostBased DSC for `MdeModulePkg` is located at +`MdeModulePkg/Test/MdeModulePkgHostTest.dsc`. + +To add automated host-based unit test building to a new package, create a +similar DSC. The new DSC should make sure to have the `NOOPT` BUILD_TARGET +and should include the line: + +``` +!include UnitTestFrameworkPkg/UnitTestFrameworkPkgHost.dsc.inc +``` + +All of the modules that are included in the `Components` section of this +DSC should be of type HOST_APPLICATION. + +### GUID Uniqueness Test - GuidCheck + +This test works on the collection of all packages rather than an individual +package. It looks at all FILE_GUIDs and GUIDs declared in DEC files and ensures +that they are unique for the codebase. This prevents, for example, accidental +duplication of GUIDs when using an existing INF as a template for a new module. + +### Cross-Package Dependency Test - DependencyCheck + +This test compares the list of all packages used in INFs files for a given +package against a list of "allowed dependencies" in plugin configuration for +that package. Any module that depends on a disallowed package will cause a test +failure. + +### Library Declaration Test - LibraryClassCheck + +This test scans at all library header files found in the `Library` folders in +all of the package's declared include directories and ensures that all files +have a matching LibraryClass declaration in the DEC file for the package. Any +missing declarations will cause a failure. + +### Invalid Character Test - CharEncodingCheck + +This test scans all files in a package to make sure that there are no invalid +Unicode characters that may cause build errors in some character +sets/localizations. + +### Spell Checking - cspell + +This test runs a spell checker on all files within the package. This is done +using the NodeJs cspell tool. For details check `.pytool/Plugin/SpellCheck`. +For this plugin to run during ci you must install nodejs and cspell and have +both available to the command line when running your CI. + +Install + +* Install nodejs from https://nodejs.org/en/ +* Install cspell + 1. Open cmd prompt with access to node and npm + 2. Run `npm install -g cspell` + + More cspell info: https://github.com/streetsidesoftware/cspell + +### License Checking - LicenseCheck + +Scans all new added files in a package to make sure code is contributed under +BSD-2-Clause-Patent. + +### Ecc tool - EccCheck + +Run the Ecc tool on the package. The Ecc tool is available in the BaseTools +package. It checks that the code complies to the EDKII coding standard. + +## PyTool Scopes + +Scopes are how the PyTool ext_dep, path_env, and plugins are activated. Meaning +that if an invocable process has a scope active then those ext_dep and path_env +will be active. To allow easy integration of PyTools capabilities there are a +few standard scopes. + +| Scope | Invocable | Description | +| :---- | :----- | :---- | +| global | edk2_invocable++ - should be base_abstract_invocable | Running an invocables | +| global-win | edk2_invocable++ | Running on Microsoft Windows | +| global-nix | edk2_invocable++ | Running on Linux based OS | +| edk2-build | | This indicates that an invocable is building EDK2 based UEFI code | +| cibuild | set in .pytool/CISettings.py | Suggested target for edk2 continuous integration builds. Tools used for CiBuilds can use this scope. Example: asl compiler | +| host-based-test | set in .pytool/CISettings.py | Turns on the host based tests and plugin | +| host-test-win | set in .pytool/CISettings.py | Enables the host based test runner for Windows | + +## Future investments + +* PatchCheck tests as plugins +* MacOS/xcode support +* Clang/LLVM support +* Visual Studio AARCH64 and ARM support +* BaseTools C tools CI/PR and binary release process +* BaseTools Python tools CI/PR process +* Extensible private/closed source platform reporting +* UEFI SCTs +* Other automation |