diff options
Diffstat (limited to 'fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/build-scripts/build_llvm.py')
| -rwxr-xr-x | fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/build-scripts/build_llvm.py | 307 |
1 files changed, 307 insertions, 0 deletions
diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/build-scripts/build_llvm.py b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/build-scripts/build_llvm.py new file mode 100755 index 000000000..3957f4b89 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/build-scripts/build_llvm.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +import argparse +import os +import pathlib +import requests +import shlex +import shutil +import subprocess +import sysconfig +import sys + + +def clone_llvm(dst_dir, llvm_repo, llvm_branch): + """ + any error will raise CallProcessError + """ + llvm_dir = dst_dir.joinpath("llvm").resolve() + + if not llvm_dir.exists(): + GIT_CLONE_CMD = f"git clone --depth 1 --branch {llvm_branch} {llvm_repo} llvm" + print(GIT_CLONE_CMD) + subprocess.check_output(shlex.split(GIT_CLONE_CMD), cwd=dst_dir) + + return llvm_dir + + +def query_llvm_version(llvm_info): + github_token = os.environ['GH_TOKEN'] + owner_project = llvm_info['repo'].replace("https://github.com/", "").replace(".git", "") + url = f"https://api.github.com/repos/{owner_project}/commits/{llvm_info['branch']}" + headers = { + 'Authorization': f"Bearer {github_token}" + } + + try: + response = requests.request("GET", url, headers=headers, data={}) + response.raise_for_status() + except requests.exceptions.HTTPError as error: + print (error) # for debugging purpose + return None + + response = response.json() + return response['sha'] + + +def build_llvm(llvm_dir, platform, backends, projects, use_clang=False, extra_flags=''): + LLVM_COMPILE_OPTIONS = [ + '-DCMAKE_BUILD_TYPE:STRING="Release"', + "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON", + "-DLLVM_APPEND_VC_REV:BOOL=ON", + "-DLLVM_BUILD_EXAMPLES:BOOL=OFF", + "-DLLVM_BUILD_LLVM_DYLIB:BOOL=OFF", + "-DLLVM_BUILD_TESTS:BOOL=OFF", + "-DLLVM_CCACHE_BUILD:BOOL=ON", + "-DLLVM_ENABLE_BINDINGS:BOOL=OFF", + "-DLLVM_ENABLE_IDE:BOOL=OFF", + "-DLLVM_ENABLE_LIBEDIT=OFF", + "-DLLVM_ENABLE_TERMINFO:BOOL=OFF", + "-DLLVM_ENABLE_ZLIB:BOOL=OFF", + "-DLLVM_INCLUDE_BENCHMARKS:BOOL=OFF", + "-DLLVM_INCLUDE_DOCS:BOOL=OFF", + "-DLLVM_INCLUDE_EXAMPLES:BOOL=OFF", + "-DLLVM_INCLUDE_UTILS:BOOL=OFF", + "-DLLVM_INCLUDE_TESTS:BOOL=OFF", + "-DLLVM_BUILD_TESTS:BOOL=OFF", + "-DLLVM_OPTIMIZED_TABLEGEN:BOOL=ON", + ] + + # use clang/clang++/lld. but macos doesn't support lld + if not sys.platform.startswith("darwin") and use_clang: + if shutil.which("clang") and shutil.which("clang++") and shutil.which("lld"): + os.environ["CC"] = "clang" + os.environ["CXX"] = "clang++" + LLVM_COMPILE_OPTIONS.append('-DLLVM_USE_LINKER:STRING="lld"') + print("Use the clang toolchain") + else: + print("Can not find clang, clang++ and lld, keep using the gcc toolchain") + else: + print("Use the gcc toolchain") + + LLVM_EXTRA_COMPILE_OPTIONS = { + "arc": [ + '-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD:STRING="ARC"', + "-DLLVM_ENABLE_LIBICUUC:BOOL=OFF", + "-DLLVM_ENABLE_LIBICUDATA:BOOL=OFF", + ], + "xtensa": [ + '-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD:STRING="Xtensa"', + ], + "windows": [ + "-DCMAKE_INSTALL_PREFIX=LLVM-install", + ], + "default": [], + } + + LLVM_TARGETS_TO_BUILD = [ + '-DLLVM_TARGETS_TO_BUILD:STRING="' + ";".join(backends) + '"' + if backends + else '-DLLVM_TARGETS_TO_BUILD:STRING="AArch64;ARM;Mips;RISCV;X86"' + ] + + LLVM_PROJECTS_TO_BUILD = [ + '-DLLVM_ENABLE_PROJECTS:STRING="' + ";".join(projects) + '"' if projects else "" + ] + + # lldb project requires libxml2 + LLVM_LIBXML2_OPTION = [ + "-DLLVM_ENABLE_LIBXML2:BOOL=" + ("ON" if "lldb" in projects else "OFF") + ] + + # enabling LLVM_INCLUDE_TOOLS will increase ~300M to the final package + LLVM_INCLUDE_TOOLS_OPTION = [ + "-DLLVM_INCLUDE_TOOLS:BOOL=ON" if projects else "-DLLVM_INCLUDE_TOOLS:BOOL=OFF" + ] + + if not llvm_dir.exists(): + raise Exception(f"{llvm_dir} doesn't exist") + + build_dir = llvm_dir.joinpath( + "win32build" if "windows" == platform else "build" + ).resolve() + build_dir.mkdir(exist_ok=True) + + lib_llvm_core_library = build_dir.joinpath("lib/libLLVMCore.a").resolve() + if lib_llvm_core_library.exists(): + print( + f"It has already been fully compiled. If want to a re-build, please remove {build_dir} manually and try again" + ) + return None + + compile_options = " ".join( + LLVM_COMPILE_OPTIONS + + LLVM_LIBXML2_OPTION + + LLVM_EXTRA_COMPILE_OPTIONS.get( + platform, LLVM_EXTRA_COMPILE_OPTIONS["default"] + ) + + LLVM_TARGETS_TO_BUILD + + LLVM_PROJECTS_TO_BUILD + + LLVM_INCLUDE_TOOLS_OPTION + ) + + CONFIG_CMD = f"cmake {compile_options} {extra_flags} ../llvm" + if "windows" == platform: + if "mingw" in sysconfig.get_platform().lower(): + CONFIG_CMD += " -G'Unix Makefiles'" + else: + CONFIG_CMD += " -A x64" + else: + CONFIG_CMD += " -G'Ninja'" + subprocess.check_call(shlex.split(CONFIG_CMD), cwd=build_dir) + + BUILD_CMD = "cmake --build . --target package" + ( + " --config Release" if "windows" == platform else "" + ) + subprocess.check_call(shlex.split(BUILD_CMD), cwd=build_dir) + + return build_dir + + +def repackage_llvm(llvm_dir): + build_dir = llvm_dir.joinpath("./build").resolve() + + packs = [f for f in build_dir.glob("LLVM-*.tar.gz")] + if len(packs) > 1: + raise Exception("Find more than one LLVM-*.tar.gz") + + if not packs: + return + + llvm_package = packs[0].name + # mv build/LLVM-*.gz . + shutil.move(str(build_dir.joinpath(llvm_package).resolve()), str(llvm_dir)) + # rm -r build + shutil.rmtree(str(build_dir)) + # mkdir build + build_dir.mkdir() + # tar xf ./LLVM-*.tar.gz --strip-components=1 --directory=build + CMD = f"tar xf {llvm_dir.joinpath(llvm_package).resolve()} --strip-components=1 --directory={build_dir}" + subprocess.check_call(shlex.split(CMD), cwd=llvm_dir) + # rm ./LLVM-1*.gz + os.remove(llvm_dir.joinpath(llvm_package).resolve()) + + +def main(): + parser = argparse.ArgumentParser(description="build necessary LLVM libraries") + parser.add_argument( + "--platform", + type=str, + choices=["android", "arc", "darwin", "linux", "windows", "xtensa"], + help="identify current platform", + ) + parser.add_argument( + "--arch", + nargs="+", + type=str, + choices=[ + "AArch64", + "ARC", + "ARM", + "Mips", + "RISCV", + "WebAssembly", + "X86", + "Xtensa", + ], + help="identify LLVM supported backends, separate by space, like '--arch ARM Mips X86'", + ) + parser.add_argument( + "--project", + nargs="+", + type=str, + default="", + choices=["clang", "lldb"], + help="identify extra LLVM projects, separate by space, like '--project clang lldb'", + ) + parser.add_argument( + "--llvm-ver", + action="store_true", + help="return the version info of generated llvm libraries", + ) + parser.add_argument( + "--use-clang", + action="store_true", + help="use clang instead of gcc", + ) + parser.add_argument( + "--extra-cmake-flags", + type=str, + default="", + help="custom extra cmake flags", + ) + options = parser.parse_args() + + # if the "platform" is not identified in the command line option, + # detect it + if not options.platform: + if sys.platform.startswith("win32") or sys.platform.startswith("msys"): + platform = "windows" + elif sys.platform.startswith("darwin"): + platform = "darwin" + else: + platform = "linux" + else: + platform = options.platform + + llvm_repo_and_branch = { + "arc": { + "repo": "https://github.com/llvm/llvm-project.git", + "repo_ssh": "git@github.com:llvm/llvm-project.git", + "branch": "release/15.x", + }, + "xtensa": { + "repo": "https://github.com/espressif/llvm-project.git", + "repo_ssh": "git@github.com:espressif/llvm-project.git", + "branch": "xtensa_release_15.x", + }, + "default": { + "repo": "https://github.com/llvm/llvm-project.git", + "repo_ssh": "git@github.com:llvm/llvm-project.git", + "branch": "release/15.x", + }, + } + + # retrieve the real file + current_file = pathlib.Path(__file__) + if current_file.is_symlink(): + current_file = pathlib.Path(os.readlink(current_file)) + + current_dir = current_file.parent.resolve() + deps_dir = current_dir.joinpath("../core/deps").resolve() + + try: + llvm_info = llvm_repo_and_branch.get(platform, llvm_repo_and_branch["default"]) + + if options.llvm_ver: + commit_hash = query_llvm_version(llvm_info) + print(commit_hash) + return commit_hash is not None + + repo_addr = llvm_info["repo"] + if os.environ.get('USE_GIT_SSH') == "true": + repo_addr = llvm_info["repo_ssh"] + else: + print("To use ssh for git clone, run: export USE_GIT_SSH=true") + + llvm_dir = clone_llvm(deps_dir, repo_addr, llvm_info["branch"]) + if ( + build_llvm( + llvm_dir, platform, options.arch, options.project, options.use_clang, + options.extra_cmake_flags + ) + is not None + ): + repackage_llvm(llvm_dir) + + return True + except subprocess.CalledProcessError: + return False + + +if __name__ == "__main__": + sys.exit(0 if main() else 1) |