diff options
Diffstat (limited to 'third_party/python/wheel/wheel/macosx_libfile.py')
| -rw-r--r-- | third_party/python/wheel/wheel/macosx_libfile.py | 428 |
1 files changed, 428 insertions, 0 deletions
diff --git a/third_party/python/wheel/wheel/macosx_libfile.py b/third_party/python/wheel/wheel/macosx_libfile.py new file mode 100644 index 0000000000..39006fb079 --- /dev/null +++ b/third_party/python/wheel/wheel/macosx_libfile.py @@ -0,0 +1,428 @@ +""" +This module contains function to analyse dynamic library +headers to extract system information + +Currently only for MacOSX + +Library file on macosx system starts with Mach-O or Fat field. +This can be distinguish by first 32 bites and it is called magic number. +Proper value of magic number is with suffix _MAGIC. Suffix _CIGAM means +reversed bytes order. +Both fields can occur in two types: 32 and 64 bytes. + +FAT field inform that this library contains few version of library +(typically for different types version). It contains +information where Mach-O headers starts. + +Each section started with Mach-O header contains one library +(So if file starts with this field it contains only one version). + +After filed Mach-O there are section fields. +Each of them starts with two fields: +cmd - magic number for this command +cmdsize - total size occupied by this section information. + +In this case only sections LC_VERSION_MIN_MACOSX (for macosx 10.13 and earlier) +and LC_BUILD_VERSION (for macosx 10.14 and newer) are interesting, +because them contains information about minimal system version. + +Important remarks: +- For fat files this implementation looks for maximum number version. + It not check if it is 32 or 64 and do not compare it with currently built package. + So it is possible to false report higher version that needed. +- All structures signatures are taken form macosx header files. +- I think that binary format will be more stable than `otool` output. + and if apple introduce some changes both implementation will need to be updated. +- The system compile will set the deployment target no lower than + 11.0 for arm64 builds. For "Universal 2" builds use the x86_64 deployment + target when the arm64 target is 11.0. +""" + +import ctypes +import os +import sys + +"""here the needed const and struct from mach-o header files""" + +FAT_MAGIC = 0xcafebabe +FAT_CIGAM = 0xbebafeca +FAT_MAGIC_64 = 0xcafebabf +FAT_CIGAM_64 = 0xbfbafeca +MH_MAGIC = 0xfeedface +MH_CIGAM = 0xcefaedfe +MH_MAGIC_64 = 0xfeedfacf +MH_CIGAM_64 = 0xcffaedfe + +LC_VERSION_MIN_MACOSX = 0x24 +LC_BUILD_VERSION = 0x32 + +CPU_TYPE_ARM64 = 0x0100000c + +mach_header_fields = [ + ("magic", ctypes.c_uint32), ("cputype", ctypes.c_int), + ("cpusubtype", ctypes.c_int), ("filetype", ctypes.c_uint32), + ("ncmds", ctypes.c_uint32), ("sizeofcmds", ctypes.c_uint32), + ("flags", ctypes.c_uint32) + ] +""" +struct mach_header { + uint32_t magic; /* mach magic number identifier */ + cpu_type_t cputype; /* cpu specifier */ + cpu_subtype_t cpusubtype; /* machine specifier */ + uint32_t filetype; /* type of file */ + uint32_t ncmds; /* number of load commands */ + uint32_t sizeofcmds; /* the size of all the load commands */ + uint32_t flags; /* flags */ +}; +typedef integer_t cpu_type_t; +typedef integer_t cpu_subtype_t; +""" + +mach_header_fields_64 = mach_header_fields + [("reserved", ctypes.c_uint32)] +""" +struct mach_header_64 { + uint32_t magic; /* mach magic number identifier */ + cpu_type_t cputype; /* cpu specifier */ + cpu_subtype_t cpusubtype; /* machine specifier */ + uint32_t filetype; /* type of file */ + uint32_t ncmds; /* number of load commands */ + uint32_t sizeofcmds; /* the size of all the load commands */ + uint32_t flags; /* flags */ + uint32_t reserved; /* reserved */ +}; +""" + +fat_header_fields = [("magic", ctypes.c_uint32), ("nfat_arch", ctypes.c_uint32)] +""" +struct fat_header { + uint32_t magic; /* FAT_MAGIC or FAT_MAGIC_64 */ + uint32_t nfat_arch; /* number of structs that follow */ +}; +""" + +fat_arch_fields = [ + ("cputype", ctypes.c_int), ("cpusubtype", ctypes.c_int), + ("offset", ctypes.c_uint32), ("size", ctypes.c_uint32), + ("align", ctypes.c_uint32) +] +""" +struct fat_arch { + cpu_type_t cputype; /* cpu specifier (int) */ + cpu_subtype_t cpusubtype; /* machine specifier (int) */ + uint32_t offset; /* file offset to this object file */ + uint32_t size; /* size of this object file */ + uint32_t align; /* alignment as a power of 2 */ +}; +""" + +fat_arch_64_fields = [ + ("cputype", ctypes.c_int), ("cpusubtype", ctypes.c_int), + ("offset", ctypes.c_uint64), ("size", ctypes.c_uint64), + ("align", ctypes.c_uint32), ("reserved", ctypes.c_uint32) +] +""" +struct fat_arch_64 { + cpu_type_t cputype; /* cpu specifier (int) */ + cpu_subtype_t cpusubtype; /* machine specifier (int) */ + uint64_t offset; /* file offset to this object file */ + uint64_t size; /* size of this object file */ + uint32_t align; /* alignment as a power of 2 */ + uint32_t reserved; /* reserved */ +}; +""" + +segment_base_fields = [("cmd", ctypes.c_uint32), ("cmdsize", ctypes.c_uint32)] +"""base for reading segment info""" + +segment_command_fields = [ + ("cmd", ctypes.c_uint32), ("cmdsize", ctypes.c_uint32), + ("segname", ctypes.c_char * 16), ("vmaddr", ctypes.c_uint32), + ("vmsize", ctypes.c_uint32), ("fileoff", ctypes.c_uint32), + ("filesize", ctypes.c_uint32), ("maxprot", ctypes.c_int), + ("initprot", ctypes.c_int), ("nsects", ctypes.c_uint32), + ("flags", ctypes.c_uint32), + ] +""" +struct segment_command { /* for 32-bit architectures */ + uint32_t cmd; /* LC_SEGMENT */ + uint32_t cmdsize; /* includes sizeof section structs */ + char segname[16]; /* segment name */ + uint32_t vmaddr; /* memory address of this segment */ + uint32_t vmsize; /* memory size of this segment */ + uint32_t fileoff; /* file offset of this segment */ + uint32_t filesize; /* amount to map from the file */ + vm_prot_t maxprot; /* maximum VM protection */ + vm_prot_t initprot; /* initial VM protection */ + uint32_t nsects; /* number of sections in segment */ + uint32_t flags; /* flags */ +}; +typedef int vm_prot_t; +""" + +segment_command_fields_64 = [ + ("cmd", ctypes.c_uint32), ("cmdsize", ctypes.c_uint32), + ("segname", ctypes.c_char * 16), ("vmaddr", ctypes.c_uint64), + ("vmsize", ctypes.c_uint64), ("fileoff", ctypes.c_uint64), + ("filesize", ctypes.c_uint64), ("maxprot", ctypes.c_int), + ("initprot", ctypes.c_int), ("nsects", ctypes.c_uint32), + ("flags", ctypes.c_uint32), + ] +""" +struct segment_command_64 { /* for 64-bit architectures */ + uint32_t cmd; /* LC_SEGMENT_64 */ + uint32_t cmdsize; /* includes sizeof section_64 structs */ + char segname[16]; /* segment name */ + uint64_t vmaddr; /* memory address of this segment */ + uint64_t vmsize; /* memory size of this segment */ + uint64_t fileoff; /* file offset of this segment */ + uint64_t filesize; /* amount to map from the file */ + vm_prot_t maxprot; /* maximum VM protection */ + vm_prot_t initprot; /* initial VM protection */ + uint32_t nsects; /* number of sections in segment */ + uint32_t flags; /* flags */ +}; +""" + +version_min_command_fields = segment_base_fields + \ + [("version", ctypes.c_uint32), ("sdk", ctypes.c_uint32)] +""" +struct version_min_command { + uint32_t cmd; /* LC_VERSION_MIN_MACOSX or + LC_VERSION_MIN_IPHONEOS or + LC_VERSION_MIN_WATCHOS or + LC_VERSION_MIN_TVOS */ + uint32_t cmdsize; /* sizeof(struct min_version_command) */ + uint32_t version; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */ + uint32_t sdk; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */ +}; +""" + +build_version_command_fields = segment_base_fields + \ + [("platform", ctypes.c_uint32), ("minos", ctypes.c_uint32), + ("sdk", ctypes.c_uint32), ("ntools", ctypes.c_uint32)] +""" +struct build_version_command { + uint32_t cmd; /* LC_BUILD_VERSION */ + uint32_t cmdsize; /* sizeof(struct build_version_command) plus */ + /* ntools * sizeof(struct build_tool_version) */ + uint32_t platform; /* platform */ + uint32_t minos; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */ + uint32_t sdk; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */ + uint32_t ntools; /* number of tool entries following this */ +}; +""" + + +def swap32(x): + return (((x << 24) & 0xFF000000) | + ((x << 8) & 0x00FF0000) | + ((x >> 8) & 0x0000FF00) | + ((x >> 24) & 0x000000FF)) + + +def get_base_class_and_magic_number(lib_file, seek=None): + if seek is None: + seek = lib_file.tell() + else: + lib_file.seek(seek) + magic_number = ctypes.c_uint32.from_buffer_copy( + lib_file.read(ctypes.sizeof(ctypes.c_uint32))).value + + # Handle wrong byte order + if magic_number in [FAT_CIGAM, FAT_CIGAM_64, MH_CIGAM, MH_CIGAM_64]: + if sys.byteorder == "little": + BaseClass = ctypes.BigEndianStructure + else: + BaseClass = ctypes.LittleEndianStructure + + magic_number = swap32(magic_number) + else: + BaseClass = ctypes.Structure + + lib_file.seek(seek) + return BaseClass, magic_number + + +def read_data(struct_class, lib_file): + return struct_class.from_buffer_copy(lib_file.read( + ctypes.sizeof(struct_class))) + + +def extract_macosx_min_system_version(path_to_lib): + with open(path_to_lib, "rb") as lib_file: + BaseClass, magic_number = get_base_class_and_magic_number(lib_file, 0) + if magic_number not in [FAT_MAGIC, FAT_MAGIC_64, MH_MAGIC, MH_MAGIC_64]: + return + + if magic_number in [FAT_MAGIC, FAT_CIGAM_64]: + class FatHeader(BaseClass): + _fields_ = fat_header_fields + + fat_header = read_data(FatHeader, lib_file) + if magic_number == FAT_MAGIC: + + class FatArch(BaseClass): + _fields_ = fat_arch_fields + else: + + class FatArch(BaseClass): + _fields_ = fat_arch_64_fields + + fat_arch_list = [read_data(FatArch, lib_file) for _ in range(fat_header.nfat_arch)] + + versions_list = [] + for el in fat_arch_list: + try: + version = read_mach_header(lib_file, el.offset) + if version is not None: + if el.cputype == CPU_TYPE_ARM64 and len(fat_arch_list) != 1: + # Xcode will not set the deployment target below 11.0.0 + # for the arm64 architecture. Ignore the arm64 deployment + # in fat binaries when the target is 11.0.0, that way + # the other architectures can select a lower deployment + # target. + # This is safe because there is no arm64 variant for + # macOS 10.15 or earlier. + if version == (11, 0, 0): + continue + versions_list.append(version) + except ValueError: + pass + + if len(versions_list) > 0: + return max(versions_list) + else: + return None + + else: + try: + return read_mach_header(lib_file, 0) + except ValueError: + """when some error during read library files""" + return None + + +def read_mach_header(lib_file, seek=None): + """ + This funcition parse mach-O header and extract + information about minimal system version + + :param lib_file: reference to opened library file with pointer + """ + if seek is not None: + lib_file.seek(seek) + base_class, magic_number = get_base_class_and_magic_number(lib_file) + arch = "32" if magic_number == MH_MAGIC else "64" + + class SegmentBase(base_class): + _fields_ = segment_base_fields + + if arch == "32": + + class MachHeader(base_class): + _fields_ = mach_header_fields + + else: + + class MachHeader(base_class): + _fields_ = mach_header_fields_64 + + mach_header = read_data(MachHeader, lib_file) + for _i in range(mach_header.ncmds): + pos = lib_file.tell() + segment_base = read_data(SegmentBase, lib_file) + lib_file.seek(pos) + if segment_base.cmd == LC_VERSION_MIN_MACOSX: + class VersionMinCommand(base_class): + _fields_ = version_min_command_fields + + version_info = read_data(VersionMinCommand, lib_file) + return parse_version(version_info.version) + elif segment_base.cmd == LC_BUILD_VERSION: + class VersionBuild(base_class): + _fields_ = build_version_command_fields + + version_info = read_data(VersionBuild, lib_file) + return parse_version(version_info.minos) + else: + lib_file.seek(pos + segment_base.cmdsize) + continue + + +def parse_version(version): + x = (version & 0xffff0000) >> 16 + y = (version & 0x0000ff00) >> 8 + z = (version & 0x000000ff) + return x, y, z + + +def calculate_macosx_platform_tag(archive_root, platform_tag): + """ + Calculate proper macosx platform tag basing on files which are included to wheel + + Example platform tag `macosx-10.14-x86_64` + """ + prefix, base_version, suffix = platform_tag.split('-') + base_version = tuple([int(x) for x in base_version.split(".")]) + base_version = base_version[:2] + if base_version[0] > 10: + base_version = (base_version[0], 0) + assert len(base_version) == 2 + if "MACOSX_DEPLOYMENT_TARGET" in os.environ: + deploy_target = tuple([int(x) for x in os.environ[ + "MACOSX_DEPLOYMENT_TARGET"].split(".")]) + deploy_target = deploy_target[:2] + if deploy_target[0] > 10: + deploy_target = (deploy_target[0], 0) + if deploy_target < base_version: + sys.stderr.write( + "[WARNING] MACOSX_DEPLOYMENT_TARGET is set to a lower value ({}) than the " + "version on which the Python interpreter was compiled ({}), and will be " + "ignored.\n".format('.'.join(str(x) for x in deploy_target), + '.'.join(str(x) for x in base_version)) + ) + else: + base_version = deploy_target + + assert len(base_version) == 2 + start_version = base_version + versions_dict = {} + for (dirpath, dirnames, filenames) in os.walk(archive_root): + for filename in filenames: + if filename.endswith('.dylib') or filename.endswith('.so'): + lib_path = os.path.join(dirpath, filename) + min_ver = extract_macosx_min_system_version(lib_path) + if min_ver is not None: + min_ver = min_ver[0:2] + if min_ver[0] > 10: + min_ver = (min_ver[0], 0) + versions_dict[lib_path] = min_ver + + if len(versions_dict) > 0: + base_version = max(base_version, max(versions_dict.values())) + + # macosx platform tag do not support minor bugfix release + fin_base_version = "_".join([str(x) for x in base_version]) + if start_version < base_version: + problematic_files = [k for k, v in versions_dict.items() if v > start_version] + problematic_files = "\n".join(problematic_files) + if len(problematic_files) == 1: + files_form = "this file" + else: + files_form = "these files" + error_message = \ + "[WARNING] This wheel needs a higher macOS version than {} " \ + "To silence this warning, set MACOSX_DEPLOYMENT_TARGET to at least " +\ + fin_base_version + " or recreate " + files_form + " with lower " \ + "MACOSX_DEPLOYMENT_TARGET: \n" + problematic_files + + if "MACOSX_DEPLOYMENT_TARGET" in os.environ: + error_message = error_message.format("is set in MACOSX_DEPLOYMENT_TARGET variable.") + else: + error_message = error_message.format( + "the version your Python interpreter is compiled against.") + + sys.stderr.write(error_message) + + platform_tag = prefix + "_" + fin_base_version + "_" + suffix + return platform_tag |