diff options
Diffstat (limited to 'src/VBox/Devices/EFI/Firmware/BaseTools/Source/Python/AutoGen/StrGather.py')
| -rwxr-xr-x | src/VBox/Devices/EFI/Firmware/BaseTools/Source/Python/AutoGen/StrGather.py | 630 |
1 files changed, 630 insertions, 0 deletions
diff --git a/src/VBox/Devices/EFI/Firmware/BaseTools/Source/Python/AutoGen/StrGather.py b/src/VBox/Devices/EFI/Firmware/BaseTools/Source/Python/AutoGen/StrGather.py new file mode 100755 index 00000000..895e2a75 --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/BaseTools/Source/Python/AutoGen/StrGather.py @@ -0,0 +1,630 @@ +## @file +# This file is used to parse a strings file and create or add to a string database +# file. +# +# Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR> +# SPDX-License-Identifier: BSD-2-Clause-Patent + +## +# Import Modules +# +from __future__ import absolute_import +import re +import Common.EdkLogger as EdkLogger +from Common.BuildToolError import * +from .UniClassObject import * +from io import BytesIO +from struct import pack, unpack +from Common.LongFilePathSupport import OpenLongFilePath as open + +## +# Static definitions +# +EFI_HII_SIBT_END = '0x00' +EFI_HII_SIBT_STRING_SCSU = '0x10' +EFI_HII_SIBT_STRING_SCSU_FONT = '0x11' +EFI_HII_SIBT_STRINGS_SCSU = '0x12' +EFI_HII_SIBT_STRINGS_SCSU_FONT = '0x13' +EFI_HII_SIBT_STRING_UCS2 = '0x14' +EFI_HII_SIBT_STRING_UCS2_FONT = '0x15' +EFI_HII_SIBT_STRINGS_UCS2 = '0x16' +EFI_HII_SIBT_STRINGS_UCS2_FONT = '0x17' +EFI_HII_SIBT_DUPLICATE = '0x20' +EFI_HII_SIBT_SKIP2 = '0x21' +EFI_HII_SIBT_SKIP1 = '0x22' +EFI_HII_SIBT_EXT1 = '0x30' +EFI_HII_SIBT_EXT2 = '0x31' +EFI_HII_SIBT_EXT4 = '0x32' +EFI_HII_SIBT_FONT = '0x40' + +EFI_HII_PACKAGE_STRINGS = '0x04' +EFI_HII_PACKAGE_FORM = '0x02' + +StringPackageType = EFI_HII_PACKAGE_STRINGS +StringPackageForm = EFI_HII_PACKAGE_FORM +StringBlockType = EFI_HII_SIBT_STRING_UCS2 +StringSkipType = EFI_HII_SIBT_SKIP2 + +HexHeader = '0x' + +COMMENT = '// ' +DEFINE_STR = '#define' +COMMENT_DEFINE_STR = COMMENT + DEFINE_STR +NOT_REFERENCED = 'not referenced' +COMMENT_NOT_REFERENCED = ' ' + COMMENT + NOT_REFERENCED +CHAR_ARRAY_DEFIN = 'unsigned char' +COMMON_FILE_NAME = 'Strings' +STRING_TOKEN = re.compile('STRING_TOKEN *\(([A-Z0-9_]+) *\)', re.MULTILINE | re.UNICODE) + +EFI_HII_ARRAY_SIZE_LENGTH = 4 +EFI_HII_PACKAGE_HEADER_LENGTH = 4 +EFI_HII_HDR_SIZE_LENGTH = 4 +EFI_HII_STRING_OFFSET_LENGTH = 4 +EFI_STRING_ID = 1 +EFI_STRING_ID_LENGTH = 2 +EFI_HII_LANGUAGE_WINDOW = 0 +EFI_HII_LANGUAGE_WINDOW_LENGTH = 2 +EFI_HII_LANGUAGE_WINDOW_NUMBER = 16 +EFI_HII_STRING_PACKAGE_HDR_LENGTH = EFI_HII_PACKAGE_HEADER_LENGTH + EFI_HII_HDR_SIZE_LENGTH + EFI_HII_STRING_OFFSET_LENGTH + EFI_HII_LANGUAGE_WINDOW_LENGTH * EFI_HII_LANGUAGE_WINDOW_NUMBER + EFI_STRING_ID_LENGTH + +H_C_FILE_HEADER = ['//', \ + '// DO NOT EDIT -- auto-generated file', \ + '//', \ + '// This file is generated by the StrGather utility', \ + '//'] +LANGUAGE_NAME_STRING_NAME = '$LANGUAGE_NAME' +PRINTABLE_LANGUAGE_NAME_STRING_NAME = '$PRINTABLE_LANGUAGE_NAME' + +## Convert a dec number to a hex string +# +# Convert a dec number to a formatted hex string in length digit +# The digit is set to default 8 +# The hex string starts with "0x" +# DecToHexStr(1000) is '0x000003E8' +# DecToHexStr(1000, 6) is '0x0003E8' +# +# @param Dec: The number in dec format +# @param Digit: The needed digit of hex string +# +# @retval: The formatted hex string +# +def DecToHexStr(Dec, Digit = 8): + return '0x{0:0{1}X}'.format(Dec, Digit) + +## Convert a dec number to a hex list +# +# Convert a dec number to a formatted hex list in size digit +# The digit is set to default 8 +# DecToHexList(1000) is ['0xE8', '0x03', '0x00', '0x00'] +# DecToHexList(1000, 6) is ['0xE8', '0x03', '0x00'] +# +# @param Dec: The number in dec format +# @param Digit: The needed digit of hex list +# +# @retval: A list for formatted hex string +# +def DecToHexList(Dec, Digit = 8): + Hex = '{0:0{1}X}'.format(Dec, Digit) + return ["0x" + Hex[Bit:Bit + 2] for Bit in range(Digit - 2, -1, -2)] + +## Convert a acsii string to a hex list +# +# Convert a acsii string to a formatted hex list +# AscToHexList('en-US') is ['0x65', '0x6E', '0x2D', '0x55', '0x53'] +# +# @param Ascii: The acsii string +# +# @retval: A list for formatted hex string +# +def AscToHexList(Ascii): + try: + return ['0x{0:02X}'.format(Item) for Item in Ascii] + except: + return ['0x{0:02X}'.format(ord(Item)) for Item in Ascii] + +## Create content of .h file +# +# Create content of .h file +# +# @param BaseName: The basename of strings +# @param UniObjectClass A UniObjectClass instance +# @param IsCompatibleMode Compatible mode +# @param UniGenCFlag UniString is generated into AutoGen C file when it is set to True +# +# @retval Str: A string of .h file content +# +def CreateHFileContent(BaseName, UniObjectClass, IsCompatibleMode, UniGenCFlag): + Str = [] + ValueStartPtr = 60 + Line = COMMENT_DEFINE_STR + ' ' + LANGUAGE_NAME_STRING_NAME + ' ' * (ValueStartPtr - len(DEFINE_STR + LANGUAGE_NAME_STRING_NAME)) + DecToHexStr(0, 4) + COMMENT_NOT_REFERENCED + Str = WriteLine(Str, Line) + Line = COMMENT_DEFINE_STR + ' ' + PRINTABLE_LANGUAGE_NAME_STRING_NAME + ' ' * (ValueStartPtr - len(DEFINE_STR + PRINTABLE_LANGUAGE_NAME_STRING_NAME)) + DecToHexStr(1, 4) + COMMENT_NOT_REFERENCED + Str = WriteLine(Str, Line) + UnusedStr = '' + + #Group the referred/Unused STRING token together. + for Index in range(2, len(UniObjectClass.OrderedStringList[UniObjectClass.LanguageDef[0][0]])): + StringItem = UniObjectClass.OrderedStringList[UniObjectClass.LanguageDef[0][0]][Index] + Name = StringItem.StringName + Token = StringItem.Token + Referenced = StringItem.Referenced + if Name is not None: + Line = '' + if Referenced == True: + if (ValueStartPtr - len(DEFINE_STR + Name)) <= 0: + Line = DEFINE_STR + ' ' + Name + ' ' + DecToHexStr(Token, 4) + else: + Line = DEFINE_STR + ' ' + Name + ' ' * (ValueStartPtr - len(DEFINE_STR + Name)) + DecToHexStr(Token, 4) + Str = WriteLine(Str, Line) + else: + if (ValueStartPtr - len(DEFINE_STR + Name)) <= 0: + Line = COMMENT_DEFINE_STR + ' ' + Name + ' ' + DecToHexStr(Token, 4) + COMMENT_NOT_REFERENCED + else: + Line = COMMENT_DEFINE_STR + ' ' + Name + ' ' * (ValueStartPtr - len(DEFINE_STR + Name)) + DecToHexStr(Token, 4) + COMMENT_NOT_REFERENCED + UnusedStr = WriteLine(UnusedStr, Line) + + Str.extend( UnusedStr) + + Str = WriteLine(Str, '') + if IsCompatibleMode or UniGenCFlag: + Str = WriteLine(Str, 'extern unsigned char ' + BaseName + 'Strings[];') + return "".join(Str) + +## Create a complete .h file +# +# Create a complet .h file with file header and file content +# +# @param BaseName: The basename of strings +# @param UniObjectClass A UniObjectClass instance +# @param IsCompatibleMode Compatible mode +# @param UniGenCFlag UniString is generated into AutoGen C file when it is set to True +# +# @retval Str: A string of complete .h file +# +def CreateHFile(BaseName, UniObjectClass, IsCompatibleMode, UniGenCFlag): + HFile = WriteLine('', CreateHFileContent(BaseName, UniObjectClass, IsCompatibleMode, UniGenCFlag)) + + return "".join(HFile) + +## Create a buffer to store all items in an array +# +# @param BinBuffer Buffer to contain Binary data. +# @param Array: The array need to be formatted +# +def CreateBinBuffer(BinBuffer, Array): + for Item in Array: + BinBuffer.write(pack("B", int(Item, 16))) + +## Create a formatted string all items in an array +# +# Use ',' to join each item in an array, and break an new line when reaching the width (default is 16) +# +# @param Array: The array need to be formatted +# @param Width: The line length, the default value is set to 16 +# +# @retval ArrayItem: A string for all formatted array items +# +def CreateArrayItem(Array, Width = 16): + MaxLength = Width + Index = 0 + Line = ' ' + ArrayItem = [] + + for Item in Array: + if Index < MaxLength: + Line = Line + Item + ', ' + Index = Index + 1 + else: + ArrayItem = WriteLine(ArrayItem, Line) + Line = ' ' + Item + ', ' + Index = 1 + ArrayItem = Write(ArrayItem, Line.rstrip()) + + return "".join(ArrayItem) + +## CreateCFileStringValue +# +# Create a line with string value +# +# @param Value: Value of the string +# +# @retval Str: A formatted string with string value +# + +def CreateCFileStringValue(Value): + Value = [StringBlockType] + Value + Str = WriteLine('', CreateArrayItem(Value)) + + return "".join(Str) + +## GetFilteredLanguage +# +# apply get best language rules to the UNI language code list +# +# @param UniLanguageList: language code definition list in *.UNI file +# @param LanguageFilterList: language code filter list of RFC4646 format in DSC file +# +# @retval UniLanguageListFiltered: the filtered language code +# +def GetFilteredLanguage(UniLanguageList, LanguageFilterList): + UniLanguageListFiltered = [] + # if filter list is empty, then consider there is no filter + if LanguageFilterList == []: + UniLanguageListFiltered = UniLanguageList + return UniLanguageListFiltered + for Language in LanguageFilterList: + # first check for exact match + if Language in UniLanguageList: + if Language not in UniLanguageListFiltered: + UniLanguageListFiltered.append(Language) + # find the first one with the same/equivalent primary tag + else: + if Language.find('-') != -1: + PrimaryTag = Language[0:Language.find('-')].lower() + else: + PrimaryTag = Language + + if len(PrimaryTag) == 3: + PrimaryTag = LangConvTable.get(PrimaryTag) + + for UniLanguage in UniLanguageList: + if UniLanguage.find('-') != -1: + UniLanguagePrimaryTag = UniLanguage[0:UniLanguage.find('-')].lower() + else: + UniLanguagePrimaryTag = UniLanguage + + if len(UniLanguagePrimaryTag) == 3: + UniLanguagePrimaryTag = LangConvTable.get(UniLanguagePrimaryTag) + + if PrimaryTag == UniLanguagePrimaryTag: + if UniLanguage not in UniLanguageListFiltered: + UniLanguageListFiltered.append(UniLanguage) + break + else: + # Here is rule 3 for "get best language" + # If tag is not listed in the Unicode file, the default ("en") tag should be used for that language + # for better processing, find the one that best suit for it. + DefaultTag = 'en' + if DefaultTag not in UniLanguageListFiltered: + # check whether language code with primary code equivalent with DefaultTag already in the list, if so, use that + for UniLanguage in UniLanguageList: + if UniLanguage.startswith('en-') or UniLanguage.startswith('eng-'): + if UniLanguage not in UniLanguageListFiltered: + UniLanguageListFiltered.append(UniLanguage) + break + else: + UniLanguageListFiltered.append(DefaultTag) + return UniLanguageListFiltered + + +## Create content of .c file +# +# Create content of .c file +# +# @param BaseName: The basename of strings +# @param UniObjectClass A UniObjectClass instance +# @param IsCompatibleMode Compatible mode +# @param UniBinBuffer UniBinBuffer to contain UniBinary data. +# @param FilterInfo Platform language filter information +# +# @retval Str: A string of .c file content +# +def CreateCFileContent(BaseName, UniObjectClass, IsCompatibleMode, UniBinBuffer, FilterInfo): + # + # Init array length + # + TotalLength = EFI_HII_ARRAY_SIZE_LENGTH + Str = '' + Offset = 0 + + EDK2Module = FilterInfo[0] + if EDK2Module: + LanguageFilterList = FilterInfo[1] + else: + # EDK module is using ISO639-2 format filter, convert to the RFC4646 format + LanguageFilterList = [LangConvTable.get(F.lower()) for F in FilterInfo[1]] + + UniLanguageList = [] + for IndexI in range(len(UniObjectClass.LanguageDef)): + UniLanguageList += [UniObjectClass.LanguageDef[IndexI][0]] + + UniLanguageListFiltered = GetFilteredLanguage(UniLanguageList, LanguageFilterList) + + + # + # Create lines for each language's strings + # + for IndexI in range(len(UniObjectClass.LanguageDef)): + Language = UniObjectClass.LanguageDef[IndexI][0] + if Language not in UniLanguageListFiltered: + continue + + StringBuffer = BytesIO() + StrStringValue = '' + ArrayLength = 0 + NumberOfUseOtherLangDef = 0 + Index = 0 + for IndexJ in range(1, len(UniObjectClass.OrderedStringList[UniObjectClass.LanguageDef[IndexI][0]])): + Item = UniObjectClass.OrderedStringListByToken[Language][IndexJ] + + Name = Item.StringName + Value = Item.StringValueByteList + Referenced = Item.Referenced + Token = Item.Token + UseOtherLangDef = Item.UseOtherLangDef + + if UseOtherLangDef != '' and Referenced: + NumberOfUseOtherLangDef = NumberOfUseOtherLangDef + 1 + Index = Index + 1 + else: + if NumberOfUseOtherLangDef > 0: + StrStringValue = WriteLine(StrStringValue, CreateArrayItem([StringSkipType] + DecToHexList(NumberOfUseOtherLangDef, 4))) + CreateBinBuffer (StringBuffer, ([StringSkipType] + DecToHexList(NumberOfUseOtherLangDef, 4))) + NumberOfUseOtherLangDef = 0 + ArrayLength = ArrayLength + 3 + if Referenced and Item.Token > 0: + Index = Index + 1 + StrStringValue = WriteLine(StrStringValue, "// %s: %s:%s" % (DecToHexStr(Index, 4), Name, DecToHexStr(Token, 4))) + StrStringValue = Write(StrStringValue, CreateCFileStringValue(Value)) + CreateBinBuffer (StringBuffer, [StringBlockType] + Value) + ArrayLength = ArrayLength + Item.Length + 1 # 1 is for the length of string type + + # + # EFI_HII_PACKAGE_HEADER + # + Offset = EFI_HII_STRING_PACKAGE_HDR_LENGTH + len(Language) + 1 + ArrayLength = Offset + ArrayLength + 1 + + # + # Create PACKAGE HEADER + # + Str = WriteLine(Str, '// PACKAGE HEADER\n') + TotalLength = TotalLength + ArrayLength + + List = DecToHexList(ArrayLength, 6) + \ + [StringPackageType] + \ + DecToHexList(Offset) + \ + DecToHexList(Offset) + \ + DecToHexList(EFI_HII_LANGUAGE_WINDOW, EFI_HII_LANGUAGE_WINDOW_LENGTH * 2) * EFI_HII_LANGUAGE_WINDOW_NUMBER + \ + DecToHexList(EFI_STRING_ID, 4) + \ + AscToHexList(Language) + \ + DecToHexList(0, 2) + Str = WriteLine(Str, CreateArrayItem(List, 16) + '\n') + + # + # Create PACKAGE DATA + # + Str = WriteLine(Str, '// PACKAGE DATA\n') + Str = Write(Str, StrStringValue) + + # + # Add an EFI_HII_SIBT_END at last + # + Str = WriteLine(Str, ' ' + EFI_HII_SIBT_END + ",") + + # + # Create binary UNI string + # + if UniBinBuffer: + CreateBinBuffer (UniBinBuffer, List) + UniBinBuffer.write (StringBuffer.getvalue()) + UniBinBuffer.write (pack("B", int(EFI_HII_SIBT_END, 16))) + StringBuffer.close() + + # + # Create line for string variable name + # "unsigned char $(BaseName)Strings[] = {" + # + AllStr = WriteLine('', CHAR_ARRAY_DEFIN + ' ' + BaseName + COMMON_FILE_NAME + '[] = {\n') + + if IsCompatibleMode: + # + # Create FRAMEWORK_EFI_HII_PACK_HEADER in compatible mode + # + AllStr = WriteLine(AllStr, '// FRAMEWORK PACKAGE HEADER Length') + AllStr = WriteLine(AllStr, CreateArrayItem(DecToHexList(TotalLength + 2)) + '\n') + AllStr = WriteLine(AllStr, '// FRAMEWORK PACKAGE HEADER Type') + AllStr = WriteLine(AllStr, CreateArrayItem(DecToHexList(2, 4)) + '\n') + else: + # + # Create whole array length in UEFI mode + # + AllStr = WriteLine(AllStr, '// STRGATHER_OUTPUT_HEADER') + AllStr = WriteLine(AllStr, CreateArrayItem(DecToHexList(TotalLength)) + '\n') + + # + # Join package data + # + AllStr = Write(AllStr, Str) + + return "".join(AllStr) + +## Create end of .c file +# +# Create end of .c file +# +# @retval Str: A string of .h file end +# +def CreateCFileEnd(): + Str = Write('', '};') + return Str + +## Create a .c file +# +# Create a complete .c file +# +# @param BaseName: The basename of strings +# @param UniObjectClass A UniObjectClass instance +# @param IsCompatibleMode Compatible Mode +# @param FilterInfo Platform language filter information +# +# @retval CFile: A string of complete .c file +# +def CreateCFile(BaseName, UniObjectClass, IsCompatibleMode, FilterInfo): + CFile = '' + CFile = WriteLine(CFile, CreateCFileContent(BaseName, UniObjectClass, IsCompatibleMode, None, FilterInfo)) + CFile = WriteLine(CFile, CreateCFileEnd()) + return "".join(CFile) + +## GetFileList +# +# Get a list for all files +# +# @param IncludeList: A list of all path to be searched +# @param SkipList: A list of all types of file could be skipped +# +# @retval FileList: A list of all files found +# +def GetFileList(SourceFileList, IncludeList, SkipList): + if IncludeList is None: + EdkLogger.error("UnicodeStringGather", AUTOGEN_ERROR, "Include path for unicode file is not defined") + + FileList = [] + if SkipList is None: + SkipList = [] + + for File in SourceFileList: + for Dir in IncludeList: + if not os.path.exists(Dir): + continue + File = os.path.join(Dir, File.Path) + # + # Ignore Dir + # + if os.path.isfile(File) != True: + continue + # + # Ignore file listed in skip list + # + IsSkip = False + for Skip in SkipList: + if os.path.splitext(File)[1].upper() == Skip.upper(): + EdkLogger.verbose("Skipped %s for string token uses search" % File) + IsSkip = True + break + + if not IsSkip: + FileList.append(File) + + break + + return FileList + +## SearchString +# +# Search whether all string defined in UniObjectClass are referenced +# All string used should be set to Referenced +# +# @param UniObjectClass: Input UniObjectClass +# @param FileList: Search path list +# @param IsCompatibleMode Compatible Mode +# +# @retval UniObjectClass: UniObjectClass after searched +# +def SearchString(UniObjectClass, FileList, IsCompatibleMode): + if FileList == []: + return UniObjectClass + + for File in FileList: + try: + if os.path.isfile(File): + Lines = open(File, 'r') + for Line in Lines: + for StrName in STRING_TOKEN.findall(Line): + EdkLogger.debug(EdkLogger.DEBUG_5, "Found string identifier: " + StrName) + UniObjectClass.SetStringReferenced(StrName) + except: + EdkLogger.error("UnicodeStringGather", AUTOGEN_ERROR, "SearchString: Error while processing file", File=File, RaiseError=False) + raise + + UniObjectClass.ReToken() + + return UniObjectClass + +## GetStringFiles +# +# This function is used for UEFI2.1 spec +# +# +def GetStringFiles(UniFilList, SourceFileList, IncludeList, IncludePathList, SkipList, BaseName, IsCompatibleMode = False, ShellMode = False, UniGenCFlag = True, UniGenBinBuffer = None, FilterInfo = [True, []]): + if len(UniFilList) > 0: + if ShellMode: + # + # support ISO 639-2 codes in .UNI files of EDK Shell + # + Uni = UniFileClassObject(sorted(UniFilList, key=lambda x: x.File), True, IncludePathList) + else: + Uni = UniFileClassObject(sorted(UniFilList, key=lambda x: x.File), IsCompatibleMode, IncludePathList) + else: + EdkLogger.error("UnicodeStringGather", AUTOGEN_ERROR, 'No unicode files given') + + FileList = GetFileList(SourceFileList, IncludeList, SkipList) + + Uni = SearchString(Uni, sorted (FileList), IsCompatibleMode) + + HFile = CreateHFile(BaseName, Uni, IsCompatibleMode, UniGenCFlag) + CFile = None + if IsCompatibleMode or UniGenCFlag: + CFile = CreateCFile(BaseName, Uni, IsCompatibleMode, FilterInfo) + if UniGenBinBuffer: + CreateCFileContent(BaseName, Uni, IsCompatibleMode, UniGenBinBuffer, FilterInfo) + + return HFile, CFile + +# +# Write an item +# +def Write(Target, Item): + if isinstance(Target,str): + Target = [Target] + if not Target: + Target = [] + if isinstance(Item,list): + Target.extend(Item) + else: + Target.append(Item) + return Target + +# +# Write an item with a break line +# +def WriteLine(Target, Item): + if isinstance(Target,str): + Target = [Target] + if not Target: + Target = [] + if isinstance(Item, list): + Target.extend(Item) + else: + Target.append(Item) + Target.append('\n') + return Target + +# This acts like the main() function for the script, unless it is 'import'ed into another +# script. +if __name__ == '__main__': + EdkLogger.info('start') + + UniFileList = [ + r'C:\\Edk\\Strings2.uni', + r'C:\\Edk\\Strings.uni' + ] + + SrcFileList = [] + for Root, Dirs, Files in os.walk('C:\\Edk'): + for File in Files: + SrcFileList.append(File) + + IncludeList = [ + r'C:\\Edk' + ] + + SkipList = ['.inf', '.uni'] + BaseName = 'DriverSample' + (h, c) = GetStringFiles(UniFileList, SrcFileList, IncludeList, SkipList, BaseName, True) + hfile = open('unistring.h', 'w') + cfile = open('unistring.c', 'w') + hfile.write(h) + cfile.write(c) + + EdkLogger.info('end') |