diff options
Diffstat (limited to 'third_party/libwebrtc/build/toolchain/win/midl.py')
| -rw-r--r-- | third_party/libwebrtc/build/toolchain/win/midl.py | 487 |
1 files changed, 487 insertions, 0 deletions
diff --git a/third_party/libwebrtc/build/toolchain/win/midl.py b/third_party/libwebrtc/build/toolchain/win/midl.py new file mode 100644 index 0000000000..cfb4220133 --- /dev/null +++ b/third_party/libwebrtc/build/toolchain/win/midl.py @@ -0,0 +1,487 @@ +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from __future__ import division +from __future__ import print_function + +import array +import difflib +import distutils.dir_util +import filecmp +import io +import operator +import os +import posixpath +import re +import shutil +import struct +import subprocess +import sys +import tempfile +import uuid + +from functools import reduce + + +def ZapTimestamp(filename): + contents = open(filename, 'rb').read() + # midl.exe writes timestamp 2147483647 (2^31 - 1) as creation date into its + # outputs, but using the local timezone. To make the output timezone- + # independent, replace that date with a fixed string of the same length. + # Also blank out the minor version number. + if filename.endswith('.tlb'): + # See https://chromium-review.googlesource.com/c/chromium/src/+/693223 for + # a fairly complete description of the .tlb binary format. + # TLB files start with a 54 byte header. Offset 0x20 stores how many types + # are defined in the file, and the header is followed by that many uint32s. + # After that, 15 section headers appear. Each section header is 16 bytes, + # starting with offset and length uint32s. + # Section 12 in the file contains custom() data. custom() data has a type + # (int, string, etc). Each custom data chunk starts with a uint16_t + # describing its type. Type 8 is string data, consisting of a uint32_t + # len, followed by that many data bytes, followed by 'W' bytes to pad to a + # 4 byte boundary. Type 0x13 is uint32 data, followed by 4 data bytes, + # followed by two 'W' to pad to a 4 byte boundary. + # The custom block always starts with one string containing "Created by + # MIDL version 8...", followed by one uint32 containing 0x7fffffff, + # followed by another uint32 containing the MIDL compiler version (e.g. + # 0x0801026e for v8.1.622 -- 0x26e == 622). These 3 fields take 0x54 bytes. + # There might be more custom data after that, but these 3 blocks are always + # there for file-level metadata. + # All data is little-endian in the file. + assert contents[0:8] == b'MSFT\x02\x00\x01\x00' + ntypes, = struct.unpack_from('<I', contents, 0x20) + custom_off, custom_len = struct.unpack_from( + '<II', contents, 0x54 + 4*ntypes + 11*16) + assert custom_len >= 0x54 + # First: Type string (0x8), followed by 0x3e characters. + assert contents[custom_off:custom_off + 6] == b'\x08\x00\x3e\x00\x00\x00' + assert re.match( + br'Created by MIDL version 8\.\d\d\.\d{4} ' + br'at ... Jan 1. ..:..:.. 2038\n', + contents[custom_off + 6:custom_off + 6 + 0x3e]) + # Second: Type uint32 (0x13) storing 0x7fffffff (followed by WW / 0x57 pad) + assert contents[custom_off+6+0x3e:custom_off+6+0x3e+8] == \ + b'\x13\x00\xff\xff\xff\x7f\x57\x57' + # Third: Type uint32 (0x13) storing MIDL compiler version. + assert contents[custom_off + 6 + 0x3e + 8:custom_off + 6 + 0x3e + 8 + + 2] == b'\x13\x00' + # Replace "Created by" string with fixed string, and fixed MIDL version with + # 8.1.622 always. + contents = ( + contents[0:custom_off + 6] + + b'Created by MIDL version 8.xx.xxxx at a redacted point in time\n' + + # uint32 (0x13) val 0x7fffffff, WW, uint32 (0x13), val 0x0801026e, WW + b'\x13\x00\xff\xff\xff\x7f\x57\x57\x13\x00\x6e\x02\x01\x08\x57\x57' + + contents[custom_off + 0x54:]) + else: + contents = re.sub( + br'File created by MIDL compiler version 8\.\d\d\.\d{4} \*/\r\n' + br'/\* at ... Jan 1. ..:..:.. 2038', + br'File created by MIDL compiler version 8.xx.xxxx */\r\n' + br'/* at a redacted point in time', contents) + contents = re.sub( + br' Oicf, W1, Zp8, env=(.....) \(32b run\), ' + br'target_arch=(AMD64|X86) 8\.\d\d\.\d{4}', + br' Oicf, W1, Zp8, env=\1 (32b run), target_arch=\2 8.xx.xxxx', + contents) + # TODO(thakis): If we need more hacks than these, try to verify checked-in + # outputs when we're using the hermetic toolchain. + # midl.exe older than 8.1.622 omit '//' after #endif, fix that: + contents = contents.replace(b'#endif !_MIDL_USE_GUIDDEF_', + b'#endif // !_MIDL_USE_GUIDDEF_') + # midl.exe puts the midl version into code in one place. To have + # predictable output, lie about the midl version if it's not 8.1.622. + # This is unfortunate, but remember that there's beauty too in imperfection. + contents = contents.replace(b'0x801026c, /* MIDL Version 8.1.620 */', + b'0x801026e, /* MIDL Version 8.1.622 */') + open(filename, 'wb').write(contents) + + +def get_tlb_contents(tlb_file): + # See ZapTimestamp() for a short overview of the .tlb format. + contents = open(tlb_file, 'rb').read() + assert contents[0:8] == b'MSFT\x02\x00\x01\x00' + ntypes, = struct.unpack_from('<I', contents, 0x20) + type_off, type_len = struct.unpack_from('<II', contents, 0x54 + 4*ntypes) + + guid_off, guid_len = struct.unpack_from( + '<II', contents, 0x54 + 4*ntypes + 5*16) + assert guid_len % 24 == 0 + + contents = array.array('B', contents) + + return contents, ntypes, type_off, guid_off, guid_len + + +def recreate_guid_hashtable(contents, ntypes, guid_off, guid_len): + # This function is called after changing guids in section 6 (the "guid" + # section). This function recreates the GUID hashtable in section 5. Since the + # hash table uses chaining, it's easiest to recompute it from scratch rather + # than trying to patch it up. + hashtab = [0xffffffff] * (0x80 // 4) + for guidind in range(guid_off, guid_off + guid_len, 24): + guidbytes, typeoff, nextguid = struct.unpack_from( + '<16sII', contents, guidind) + words = struct.unpack('<8H', guidbytes) + # midl seems to use the following simple hash function for GUIDs: + guidhash = reduce(operator.xor, [w for w in words]) % (0x80 // 4) + nextguid = hashtab[guidhash] + struct.pack_into('<I', contents, guidind + 0x14, nextguid) + hashtab[guidhash] = guidind - guid_off + hash_off, hash_len = struct.unpack_from( + '<II', contents, 0x54 + 4*ntypes + 4*16) + for i, hashval in enumerate(hashtab): + struct.pack_into('<I', contents, hash_off + 4*i, hashval) + + +def overwrite_guids_h(h_file, dynamic_guids): + contents = open(h_file, 'rb').read() + for key in dynamic_guids: + contents = re.sub(key, dynamic_guids[key], contents, flags=re.I) + open(h_file, 'wb').write(contents) + + +def get_uuid_format(guid, prefix): + formatted_uuid = b'0x%s,0x%s,0x%s,' % (guid[0:8], guid[9:13], guid[14:18]) + formatted_uuid += b'%s0x%s,0x%s' % (prefix, guid[19:21], guid[21:23]) + for i in range(24, len(guid), 2): + formatted_uuid += b',0x' + guid[i:i + 2] + return formatted_uuid + + +def get_uuid_format_iid_file(guid): + # Convert from "D0E1CACC-C63C-4192-94AB-BF8EAD0E3B83" to + # 0xD0E1CACC,0xC63C,0x4192,0x94,0xAB,0xBF,0x8E,0xAD,0x0E,0x3B,0x83. + return get_uuid_format(guid, b'') + + +def overwrite_guids_iid(iid_file, dynamic_guids): + contents = open(iid_file, 'rb').read() + for key in dynamic_guids: + contents = re.sub(get_uuid_format_iid_file(key), + get_uuid_format_iid_file(dynamic_guids[key]), + contents, + flags=re.I) + open(iid_file, 'wb').write(contents) + + +def get_uuid_format_proxy_file(guid): + # Convert from "D0E1CACC-C63C-4192-94AB-BF8EAD0E3B83" to + # {0xD0E1CACC,0xC63C,0x4192,{0x94,0xAB,0xBF,0x8E,0xAD,0x0E,0x3B,0x83}}. + return get_uuid_format(guid, b'{') + + +def overwrite_guids_proxy(proxy_file, dynamic_guids): + contents = open(proxy_file, 'rb').read() + for key in dynamic_guids: + contents = re.sub(get_uuid_format_proxy_file(key), + get_uuid_format_proxy_file(dynamic_guids[key]), + contents, + flags=re.I) + open(proxy_file, 'wb').write(contents) + + +def getguid(contents, offset): + # Returns a guid string of the form "D0E1CACC-C63C-4192-94AB-BF8EAD0E3B83". + g0, g1, g2, g3 = struct.unpack_from('<IHH8s', contents, offset) + g3 = b''.join([b'%02X' % g for g in bytearray(g3)]) + return b'%08X-%04X-%04X-%s-%s' % (g0, g1, g2, g3[0:4], g3[4:]) + + +def setguid(contents, offset, guid): + guid = uuid.UUID(guid.decode('utf-8')) + struct.pack_into('<IHH8s', contents, offset, + *(guid.fields[0:3] + (guid.bytes[8:], ))) + + +def overwrite_guids_tlb(tlb_file, dynamic_guids): + contents, ntypes, type_off, guid_off, guid_len = get_tlb_contents(tlb_file) + + for i in range(0, guid_len, 24): + current_guid = getguid(contents, guid_off + i) + for key in dynamic_guids: + if key.lower() == current_guid.lower(): + setguid(contents, guid_off + i, dynamic_guids[key]) + + recreate_guid_hashtable(contents, ntypes, guid_off, guid_len) + open(tlb_file, 'wb').write(contents) + + +# Handle multiple guid substitutions, where |dynamic_guids| is of the form +# "PLACEHOLDER-GUID-158428a4-6014-4978-83ba-9fad0dabe791=" +# "3d852661-c795-4d20-9b95-5561e9a1d2d9," +# "PLACEHOLDER-GUID-63B8FFB1-5314-48C9-9C57-93EC8BC6184B=" +# "D0E1CACC-C63C-4192-94AB-BF8EAD0E3B83". +# +# Before specifying |dynamic_guids| in the build, the IDL file is first compiled +# with "158428a4-6014-4978-83ba-9fad0dabe791" and +# "63B8FFB1-5314-48C9-9C57-93EC8BC6184B". These are the "replaceable" guids, +# i.e., guids that can be replaced in future builds. The resulting MIDL outputs +# are copied over to src\third_party\win_build_output\. +# +# Then, in the future, any changes to these guids can be accomplished by +# providing |dynamic_guids| of the format above in the build file. These +# "dynamic" guid changes by themselves will not require the MIDL compiler and +# therefore will not require copying output over to +# src\third_party\win_build_output\. +# +# The pre-generated src\third_party\win_build_output\ files are used for +# cross-compiling on other platforms, since the MIDL compiler is Windows-only. +def overwrite_guids(h_file, iid_file, proxy_file, tlb_file, dynamic_guids): + # Fix up GUIDs in .h, _i.c, _p.c, and .tlb. + overwrite_guids_h(h_file, dynamic_guids) + overwrite_guids_iid(iid_file, dynamic_guids) + overwrite_guids_proxy(proxy_file, dynamic_guids) + if tlb_file: + overwrite_guids_tlb(tlb_file, dynamic_guids) + + +# This function removes all occurrences of 'PLACEHOLDER-GUID-' from the +# template, and if |dynamic_guids| is specified, also replaces the guids within +# the file. Finally, it writes the resultant output to the |idl| file. +def generate_idl_from_template(idl_template, dynamic_guids, idl): + contents = open(idl_template, 'rb').read() + contents = re.sub(b'PLACEHOLDER-GUID-', b'', contents, flags=re.I) + if dynamic_guids: + for key in dynamic_guids: + contents = re.sub(key, dynamic_guids[key], contents, flags=re.I) + open(idl, 'wb').write(contents) + + +# This function runs the MIDL compiler with the provided arguments. It creates +# and returns a tuple of |0,midl_output_dir| on success. +def run_midl(args, env_dict): + midl_output_dir = tempfile.mkdtemp() + delete_midl_output_dir = True + + try: + popen = subprocess.Popen(args + ['/out', midl_output_dir], + shell=True, + universal_newlines=True, + env=env_dict, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + out, _ = popen.communicate() + if popen.returncode != 0: + return popen.returncode, midl_output_dir + + # Filter junk out of stdout, and write filtered versions. Output we want + # to filter is pairs of lines that look like this: + # Processing C:\Program Files (x86)\Microsoft SDKs\...\include\objidl.idl + # objidl.idl + lines = out.splitlines() + prefixes = ('Processing ', '64 bit Processing ') + processing = set( + os.path.basename(x) for x in lines if x.startswith(prefixes)) + for line in lines: + if not line.startswith(prefixes) and line not in processing: + print(line) + + for f in os.listdir(midl_output_dir): + ZapTimestamp(os.path.join(midl_output_dir, f)) + + delete_midl_output_dir = False + finally: + if os.path.exists(midl_output_dir) and delete_midl_output_dir: + shutil.rmtree(midl_output_dir) + + return 0, midl_output_dir + + +# This function adds support for dynamic generation of guids: when values are +# specified as 'uuid5:name', this function will substitute the values with +# generated dynamic guids using the uuid5 function. The uuid5 function generates +# a guid based on the SHA-1 hash of a namespace identifier (which is the guid +# that comes after 'PLACEHOLDER-GUID-') and a name (which is a string, such as a +# version string "87.1.2.3"). +# +# For instance, when |dynamic_guid| is of the form: +# "PLACEHOLDER-GUID-158428a4-6014-4978-83ba-9fad0dabe791=uuid5:88.0.4307.0 +# ," +# "PLACEHOLDER-GUID-63B8FFB1-5314-48C9-9C57-93EC8BC6184B=uuid5:88.0.4307.0 +# " +# +# "PLACEHOLDER-GUID-158428a4-6014-4978-83ba-9fad0dabe791" would be substituted +# with uuid5("158428a4-6014-4978-83ba-9fad0dabe791", "88.0.4307.0"), which is +# "64700170-AD80-5DE3-924E-2F39D862CFD5". And +# "PLACEHOLDER-GUID-63B8FFB1-5314-48C9-9C57-93EC8BC6184B" would be +# substituted with uuid5("63B8FFB1-5314-48C9-9C57-93EC8BC6184B", "88.0.4307.0"), +# which is "7B6E7538-3C38-5565-BC92-42BCEE268D76". +def uuid5_substitutions(dynamic_guids): + for key, value in dynamic_guids.items(): + if value.startswith('uuid5:'): + name = value.split('uuid5:', 1)[1] + assert name + dynamic_guids[key] = str(uuid.uuid5(uuid.UUID(key), name)).upper() + + +def main(arch, gendir, outdir, dynamic_guids, tlb, h, dlldata, iid, proxy, + clang, idl, *flags): + # Copy checked-in outputs to final location. + source = gendir + if os.path.isdir(os.path.join(source, os.path.basename(idl))): + source = os.path.join(source, os.path.basename(idl)) + source = os.path.join(source, arch.split('.')[1]) # Append 'x86' or 'x64'. + source = os.path.normpath(source) + + source_exists = True + if not os.path.isdir(source): + source_exists = False + if sys.platform != 'win32': + print('Directory %s needs to be populated from Windows first' % source) + return 1 + + # This is a brand new IDL file that does not have outputs under + # third_party\win_build_output\midl. We create an empty directory for now. + os.makedirs(source) + + common_files = [h, iid] + if tlb != 'none': + # Not all projects use tlb files. + common_files += [tlb] + else: + tlb = None + + if dlldata != 'none': + # Not all projects use dlldta files. + common_files += [dlldata] + else: + dlldata = None + + # Not all projects use proxy files + if proxy != 'none': + # Not all projects use proxy files. + common_files += [proxy] + else: + proxy = None + + for source_file in common_files: + file_path = os.path.join(source, source_file) + if not os.path.isfile(file_path): + source_exists = False + if sys.platform != 'win32': + print('File %s needs to be generated from Windows first' % file_path) + return 1 + + # Either this is a brand new IDL file that does not have outputs under + # third_party\win_build_output\midl or the file is (unexpectedly) missing. + # We create an empty file for now. The rest of the machinery below will + # then generate the correctly populated file using the MIDL compiler and + # instruct the developer to copy that file under + # third_party\win_build_output\midl. + open(file_path, 'wb').close() + shutil.copy(file_path, outdir) + + if dynamic_guids != 'none': + assert '=' in dynamic_guids + if dynamic_guids.startswith("ignore_proxy_stub,"): + # TODO(ganesh): The custom proxy/stub file ("_p.c") is not generated + # correctly for dynamic IIDs (but correctly if there are only dynamic + # CLSIDs). The proxy/stub lookup functions generated by MIDL.exe within + # "_p.c" rely on a sorted set of vtable lists, which we are not currently + # regenerating. At the moment, no project in Chromium that uses dynamic + # IIDs is relying on the custom proxy/stub file. So for now, if + # |dynamic_guids| is prefixed with "ignore_proxy_stub,", we exclude the + # custom proxy/stub file from the directory comparisons. + common_files.remove(proxy) + dynamic_guids = dynamic_guids.split("ignore_proxy_stub,", 1)[1] + dynamic_guids = re.sub('PLACEHOLDER-GUID-', '', dynamic_guids, flags=re.I) + dynamic_guids = dynamic_guids.split(',') + dynamic_guids = dict(s.split('=') for s in dynamic_guids) + uuid5_substitutions(dynamic_guids) + dynamic_guids_bytes = { + k.encode('utf-8'): v.encode('utf-8') + for k, v in dynamic_guids.items() + } + if source_exists: + overwrite_guids(*(os.path.join(outdir, file) if file else None + for file in [h, iid, proxy, tlb]), + dynamic_guids=dynamic_guids_bytes) + else: + dynamic_guids = None + + # On non-Windows, that's all we can do. + if sys.platform != 'win32': + return 0 + + idl_template = None + if dynamic_guids: + idl_template = idl + + # posixpath is used here to keep the MIDL-generated files with a uniform + # separator of '/' instead of mixed '/' and '\\'. + idl = posixpath.join( + outdir, + os.path.splitext(os.path.basename(idl_template))[0] + '.idl') + + # |idl_template| can contain one or more occurrences of guids that are + # substituted with |dynamic_guids|, and then MIDL is run on the substituted + # IDL file. + generate_idl_from_template(idl_template, dynamic_guids_bytes, idl) + + # On Windows, run midl.exe on the input and check that its outputs are + # identical to the checked-in outputs (after replacing guids if + # |dynamic_guids| is specified). + + # Read the environment block from the file. This is stored in the format used + # by CreateProcess. Drop last 2 NULs, one for list terminator, one for + # trailing vs. separator. + env_pairs = open(arch).read()[:-2].split('\0') + env_dict = dict([item.split('=', 1) for item in env_pairs]) + + # Extract the /D options and send them to the preprocessor. + preprocessor_options = '-E -nologo -Wno-nonportable-include-path' + preprocessor_options += ''.join( + [' ' + flag for flag in flags if flag.startswith('/D')]) + args = ['midl', '/nologo'] + list(flags) + (['/tlb', tlb] if tlb else []) + [ + '/h', h + ] + (['/dlldata', dlldata] if dlldata else []) + ['/iid', iid] + ( + ['/proxy', proxy] if proxy else + []) + ['/cpp_cmd', clang, '/cpp_opt', preprocessor_options, idl] + + returncode, midl_output_dir = run_midl(args, env_dict) + if returncode != 0: + return returncode + + # Now compare the output in midl_output_dir to the copied-over outputs. + _, mismatch, errors = filecmp.cmpfiles(midl_output_dir, outdir, common_files) + assert not errors + + if mismatch: + print('midl.exe output different from files in %s, see %s' % + (outdir, midl_output_dir)) + for f in mismatch: + if f.endswith('.tlb'): continue + fromfile = os.path.join(outdir, f) + tofile = os.path.join(midl_output_dir, f) + print(''.join( + difflib.unified_diff( + io.open(fromfile).readlines(), + io.open(tofile).readlines(), fromfile, tofile))) + + if dynamic_guids: + # |idl_template| can contain one or more occurrences of guids prefixed + # with 'PLACEHOLDER-GUID-'. We first remove the extraneous + # 'PLACEHOLDER-GUID-' prefix and then run MIDL on the substituted IDL + # file. + # No guid substitutions are done at this point, because we want to compile + # with the placeholder guids and then instruct the user to copy the output + # over to |source| which is typically src\third_party\win_build_output\. + # In future runs, the placeholder guids in |source| are replaced with the + # guids specified in |dynamic_guids|. + generate_idl_from_template(idl_template, None, idl) + returncode, midl_output_dir = run_midl(args, env_dict) + if returncode != 0: + return returncode + + print('To rebaseline:') + print(r' copy /y %s\* %s' % (midl_output_dir, source)) + return 1 + + return 0 + + +if __name__ == '__main__': + sys.exit(main(*sys.argv[1:])) |