diff options
Diffstat (limited to 'gfx/angle/update-angle.py')
| -rwxr-xr-x | gfx/angle/update-angle.py | 560 |
1 files changed, 560 insertions, 0 deletions
diff --git a/gfx/angle/update-angle.py b/gfx/angle/update-angle.py new file mode 100755 index 0000000000..348a1cf745 --- /dev/null +++ b/gfx/angle/update-angle.py @@ -0,0 +1,560 @@ +#! /usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +assert __name__ == "__main__" + +""" +To update ANGLE in Gecko, use Windows with git-bash, and setup depot_tools, python2, and +python3. Because depot_tools expects `python` to be `python2` (shame!), python2 must come +before python3 in your path. + +Upstream: https://chromium.googlesource.com/angle/angle + +Our repo: https://github.com/mozilla/angle +It has branches like 'firefox-60' which is the branch we use for pulling into +Gecko with this script. + +This script leaves a record of the merge-base and cherry-picks that we pull into +Gecko. (gfx/angle/cherries.log) + +ANGLE<->Chrome version mappings are here: https://omahaproxy.appspot.com/ +An easy choice is to grab Chrome's Beta's ANGLE branch. + +## Usage + +Prepare your env: + +* If in `cmd`: + `export PATH="$PATH:/path/to/depot_tools"` +* If in `powershell`: + `$env:Path += ";C:\path\to\depot_tools"` + +If this is a new repo, don't forget: + +~~~ +# In the angle repo: +./scripts/bootstrap.py +gclient sync +~~~ + +Update: (in the angle repo) + +~~~ +# In the angle repo: +/path/to/gecko/gfx/angle/update-angle.py origin +git push moz # Push the firefox-XX branch to github.com/mozilla/angle +~~~~ + +""" + +import json +import os +import pathlib +import re +import shutil +import subprocess +import sys +from typing import * # mypy annotations + +REPO_DIR = pathlib.Path.cwd() +GECKO_ANGLE_DIR = pathlib.Path(__file__).parent + +OUT_DIR = pathlib.Path("out") + +COMMON_HEADER = [ + "# Generated by update-angle.py", + "", + 'include("../../moz.build.common")', +] + +ROOTS = ["//:translator", "//:libEGL", "//:libGLESv2"] + +CHECK_ONLY = False +args = sys.argv[1:] +while True: + arg = args.pop(0) + if arg == "--check": + CHECK_ONLY = True + continue + args.insert(0, arg) + break + +GN_ENV = dict(os.environ) +GN_ENV["DEPOT_TOOLS_WIN_TOOLCHAIN"] = "0" + +(GIT_REMOTE,) = args # Not always 'origin'! + +# ------------------------------------------------------------------------------ + + +def run_checked(*args, **kwargs): + print(" ", args) + sys.stdout.flush() + return subprocess.run(args, check=True, **kwargs) + + +def sorted_items(x): + for k in sorted(x.keys()): + yield (k, x[k]) + + +def collapse_dotdots(path): + split = path.split("/") + + ret = [] + for x in split: + if x == ".." and ret: + ret.pop() + continue + ret.append(x) + continue + + return "/".join(ret) + + +def dag_traverse(root_keys: Sequence[str], pre_recurse_func: Callable[[str], list]): + visited_keys: Set[str] = set() + + def recurse(key): + if key in visited_keys: + return + visited_keys.add(key) + + t = pre_recurse_func(key) + try: + (next_keys, post_recurse_func) = t + except ValueError: + (next_keys,) = t + post_recurse_func = None + + for x in next_keys: + recurse(x) + + if post_recurse_func: + post_recurse_func(key) + return + + for x in root_keys: + recurse(x) + return + + +# ------------------------------------------------------------------------------ + +print("Importing graph") + +# shutil.rmtree(str(OUT_DIR), True) +OUT_DIR.mkdir(exist_ok=True) + +GN_ARGS = b""" +# Build arguments go here. +# See "gn args <out_dir> --list" for available build arguments. +is_clang = true +is_debug = false +angle_build_all = false +angle_enable_abseil = false +angle_enable_apple_translator_workarounds = true +angle_enable_essl = true +angle_enable_gl = false +angle_enable_gl_desktop_frontend = false +angle_enable_glsl = true +angle_enable_null = false +angle_enable_share_context_lock = true +angle_enable_vulkan = false +angle_has_astc_encoder = false +use_custom_libcxx = false +"""[ + 1: +] +args_gn_path = OUT_DIR / "args.gn" +args_gn_path.write_bytes(GN_ARGS) + +try: + run_checked("gn", "gen", str(OUT_DIR), shell=True, env=GN_ENV) +except subprocess.CalledProcessError: + sys.stderr.buffer.write(b"`gn` failed. Is depot_tools in your PATH?\n") + exit(1) + +p = run_checked( + "python3", + "scripts/export_targets.py", + str(OUT_DIR), + *ROOTS, + stdout=subprocess.PIPE, + shell=True, + env=GN_ENV, +) + +# - + +print("\nProcessing graph") +libraries = json.loads(p.stdout.decode()) + +# - +# HACKHACKHACK: Inject linux/mac sources instead of trying to merge graphs of different +# platforms. +# descs["//:angle_common"]["sources"] += +EXTRA_ANGLE_COMMON_SOURCES = [ + "//src/common/system_utils_apple.cpp", + "//src/common/system_utils_linux.cpp", + "//src/common/system_utils_mac.cpp", + "//src/common/system_utils_posix.cpp", +] + +angle_common = libraries["//:angle_common"] +angle_common["sources"] += EXTRA_ANGLE_COMMON_SOURCES +angle_common["sources"] = sorted(angle_common["sources"]) + +# - +# Reuse our own zlib + +del libraries["//third_party/zlib:zlib"] + +# - + +if CHECK_ONLY: + print("\n--check complete.") + exit(0) + +# ------------------------------------------------------------------------------ +# Output to moz.builds + +import vendor_from_git + +print("") +vendor_from_git.record_cherry_picks(GECKO_ANGLE_DIR, GIT_REMOTE) + +# -- + + +def sortedi(x): + return sorted(x, key=str.lower) + + +def append_arr(dest, name, vals, indent=0): + if not vals: + return + + dest.append("{}{} += [".format(" " * 4 * indent, name)) + for x in sortedi(vals): + dest.append('{}"{}",'.format(" " * 4 * (indent + 1), x)) + dest.append("{}]".format(" " * 4 * indent)) + dest.append("") + return + + +REGISTERED_DEFINES = { + "ADLER32_SIMD_SSSE3": False, + "ANGLE_CAPTURE_ENABLED": True, + "ANGLE_DISABLE_POOL_ALLOC": True, + "ANGLE_EGL_LIBRARY_NAME": False, + "ANGLE_ENABLE_APPLE_WORKAROUNDS": True, + "ANGLE_ENABLE_D3D11": True, + "ANGLE_ENABLE_D3D11_COMPOSITOR_NATIVE_WINDOW": True, + "ANGLE_ENABLE_D3D9": True, + "ANGLE_ENABLE_DEBUG_ANNOTATIONS": True, + "ANGLE_ENABLE_NULL": False, + "ANGLE_ENABLE_OPENGL": False, + "ANGLE_ENABLE_OPENGL_NULL": False, + "ANGLE_ENABLE_ESSL": True, + "ANGLE_ENABLE_GLSL": True, + "ANGLE_ENABLE_HLSL": True, + "ANGLE_ENABLE_SHARE_CONTEXT_LOCK": True, + "ANGLE_GENERATE_SHADER_DEBUG_INFO": True, + "ANGLE_GLESV2_LIBRARY_NAME": True, + "ANGLE_HAS_ASTCENC": True, + "ANGLE_HAS_VULKAN_SYSTEM_INFO": False, + "ANGLE_IS_64_BIT_CPU": False, + "ANGLE_IS_WIN": False, + "ANGLE_PRELOADED_D3DCOMPILER_MODULE_NAMES": False, + "ANGLE_SHARED_LIBVULKAN": True, + "ANGLE_USE_CUSTOM_LIBVULKAN": True, + "ANGLE_USE_EGL_LOADER": True, + "ANGLE_VK_LAYERS_DIR": True, + "ANGLE_VK_MOCK_ICD_JSON": True, + "ANGLE_VMA_VERSION": True, + "ASTCENC_DECOMPRESS_ONLY": True, + "CERT_CHAIN_PARA_HAS_EXTRA_FIELDS": False, + "CHROMIUM_BUILD": False, + "COMPONENT_BUILD": False, + "CRC32_SIMD_SSE42_PCLMUL": False, + "DEFLATE_FILL_WINDOW_SSE2": False, + "DYNAMIC_ANNOTATIONS_ENABLED": True, + "EGL_EGL_PROTOTYPES": True, + "EGL_EGLEXT_PROTOTYPES": True, + "EGLAPI": True, + "FIELDTRIAL_TESTING_ENABLED": False, + "FULL_SAFE_BROWSING": False, + "GL_API": True, + "GL_APICALL": True, + "GL_GLES_PROTOTYPES": True, + "GL_GLEXT_PROTOTYPES": True, + "GPU_INFO_USE_SETUPAPI": True, + "INFLATE_CHUNK_READ_64LE": False, + "INFLATE_CHUNK_SIMD_SSE2": False, + "LIBANGLE_IMPLEMENTATION": True, + "LIBEGL_IMPLEMENTATION": True, + "LIBGLESV2_IMPLEMENTATION": True, + "NOMINMAX": True, + "NO_TCMALLOC": False, + # Else: gfx/angle/checkout/src/libANGLE/renderer/d3d/d3d11/win32/NativeWindow11Win32.cpp(89): error C2787: 'IDCompositionDevice': no GUID has been associated with this object + "NTDDI_VERSION": False, + "PSAPI_VERSION": False, + "SAFE_BROWSING_CSD": False, + "SAFE_BROWSING_DB_LOCAL": False, + "UNICODE": True, + "USE_AURA": False, + "V8_DEPRECATION_WARNINGS": False, + "VK_USE_PLATFORM_WIN32_KHR": False, + "WIN32": False, + "WIN32_LEAN_AND_MEAN": False, + "WINAPI_FAMILY": False, + "WINVER": True, + # Otherwise: + # gfx/angle/targets/libANGLE + # In file included from c:/dev/mozilla/gecko4/gfx/angle/checkout/src/libANGLE/renderer/d3d/d3d11/converged/CompositorNativeWindow11.cpp:10: + # In file included from c:/dev/mozilla/gecko4/gfx/angle/checkout/src\libANGLE/renderer/d3d/d3d11/converged/CompositorNativeWindow11.h:17: + # C:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\winrt\Windows.ui.composition.interop.h(103,20): error: unknown type name 'POINTER_INFO' + # _In_ const POINTER_INFO& pointerInfo + # ^ + "WTF_USE_DYNAMIC_ANNOTATIONS": False, + "X86_WINDOWS": False, + "ZLIB_IMPLEMENTATION": True, + "_ATL_NO_OPENGL": True, + "_CRT_NONSTDC_NO_DEPRECATE": True, + "_CRT_NONSTDC_NO_WARNINGS": True, + "_CRT_RAND_S": True, + "_CRT_SECURE_NO_DEPRECATE": True, + "_DEBUG": False, + "_HAS_EXCEPTIONS": True, + "_HAS_ITERATOR_DEBUGGING": False, + "_SCL_SECURE_NO_DEPRECATE": True, + "_SECURE_ATL": True, + "_UNICODE": True, + "_USING_V110_SDK71_": False, + "_WIN32_WINNT": False, + "_WINDOWS": False, + "_WINSOCK_DEPRECATED_NO_WARNINGS": True, + "__STD_C": False, + # clang specific + "CR_CLANG_REVISION": True, + "NDEBUG": False, + "NVALGRIND": False, + "_HAS_NODISCARD": False, +} + +# - + +print("\nRun actions") +required_files: Set[str] = set() +required_files.add("//LICENSE") + +run_checked("ninja", "-C", str(OUT_DIR), ":angle_commit_id", shell=True) +required_files.add("//out/gen/angle/angle_commit.h") + +# - + +# Export our targets +print("\nExport targets") + +# Clear our dest directories +targets_dir = pathlib.Path(GECKO_ANGLE_DIR, "targets") +checkout_dir = pathlib.Path(GECKO_ANGLE_DIR, "checkout") + +shutil.rmtree(targets_dir, True) +shutil.rmtree(checkout_dir, True) +targets_dir.mkdir(exist_ok=True) +checkout_dir.mkdir(exist_ok=True) + +# - + +RE_TARGET_NAME = re.compile("//(.*):(.+)") + + +def export_target(target_full_name) -> Set[str]: + # print(' ', target_full_name) + descs = libraries + desc = descs[target_name] + flat = desc + + m = RE_TARGET_NAME.match(target_name) + assert m, target_name + name = m.group(2) + + required_files: Set[str] = set(flat["sources"]) + + # Create our manifest lines + target_dir = targets_dir / name + target_dir.mkdir(exist_ok=True) + + lines = list(COMMON_HEADER) + lines.append("") + + for x in sorted(set(desc["defines"])): + try: + (k, v) = x.split("=", 1) + if v.startswith('"'): + v = f"'{v}'" + else: + v = f'"{v}"' + except ValueError: + (k, v) = (x, "True") + + line = f'DEFINES["{k}"] = {v}' + try: + if REGISTERED_DEFINES[k] == False: + line = "# " + line + except KeyError: + print(f"[{name}] Unrecognized define: {k}") + line = "# Unrecognized: " + line + lines.append(line) + lines.append("") + + cxxflags = set(desc["cflags"] + desc["cflags_cc"]) + + def fixup_paths(listt): + for x in set(listt): + assert x.startswith("//"), x + yield "../../checkout/" + x[2:] + + sources_by_config: Dict[str, List[str]] = {} + extras: Dict[str, str] = dict() + for x in fixup_paths(flat["sources"]): + # print(' '*5, x) + (b, e) = x.rsplit(".", 1) + if e in ["h", "hpp", "y", "l", "inc", "inl"]: + continue + elif e in ["cpp", "cc", "c"]: + if b.endswith("_win") or b.endswith("_win32"): + config = 'CONFIG["OS_ARCH"] == "WINNT"' + elif b.endswith("_linux"): + # Include these on BSDs too. + config = 'CONFIG["OS_ARCH"] not in ("Darwin", "WINNT")' + elif b.endswith("_apple") or b.endswith("_mac"): + config = 'CONFIG["OS_ARCH"] == "Darwin"' + elif b.endswith("_posix"): + config = 'CONFIG["OS_ARCH"] != "WINNT"' + else: + config = "" # None can't compare against str. + + sources_by_config.setdefault(config, []).append(x) + continue + elif e == "rc": + assert "RCFILE" not in extras, (target_name, extras["RCFILE"], x) + extras["RCFILE"] = f'"{x}"' + continue + elif e == "def": + assert "DEFFILE" not in extras, (target_name, extras["DEFFILE"], x) + extras["DEFFILE"] = f'"{x}"' + continue + else: + assert False, ("Unhandled ext:", x) + + ldflags = set(desc["ldflags"]) + DEF_PREFIX = "/DEF:" + for x in set(ldflags): + if x.startswith(DEF_PREFIX): + def_path = x[len(DEF_PREFIX) :] + required_files.add(def_path) + assert "DEFFILE" not in extras + ldflags.remove(x) + + def_path = str(OUT_DIR) + "/" + def_path + def_path = "//" + collapse_dotdots(def_path) + + def_rel_path = list(fixup_paths([def_path]))[0] + extras["DEFFILE"] = '"{}"'.format(def_rel_path) + elif x.startswith("/PDBSourcePath:"): + ldflags.remove(x) + + os_libs = list(map(lambda x: x[: -len(".lib")], set(desc.get("libs", [])))) + + def append_arr_commented(dest, name, src): + lines = [] + append_arr(lines, name, src) + + def comment(x): + if x: + x = "# " + x + return x + + lines = map(comment, lines) + dest += lines + + append_arr(lines, "LOCAL_INCLUDES", fixup_paths(desc["include_dirs"])) + append_arr_commented(lines, "CXXFLAGS", cxxflags) + + for config, v in sorted_items(sources_by_config): + indent = 0 + if config: + lines.append("if {}:".format(config)) + indent = 1 + append_arr(lines, "SOURCES", v, indent=indent) + + dep_libs: Set[str] = set() + for dep_full_name in set(flat["dep_libs"]): + assert dep_full_name.startswith("//"), dep_name + (_, dep_name) = dep_full_name.split(":") + dep_libs.add(dep_name) + + dep_dirs = set(dep_libs) + dep_dirs.discard("zlib") + + append_arr(lines, "USE_LIBS", dep_libs) + append_arr(lines, "DIRS", ["../" + x for x in dep_dirs]) + append_arr(lines, "OS_LIBS", os_libs) + append_arr_commented(lines, "LDFLAGS", ldflags) + + for k, v in sorted(extras.items()): + lines.append("{} = {}".format(k, v)) + + lib_type = desc["type"] + if lib_type == "shared_library": + lines.append(f'GeckoSharedLibrary("{name}", linkage=None)') + elif lib_type == "static_library": + lines.append(f'Library("{name}")') + else: + assert False, lib_type + + lines.append("") + # EOF newline + + # Write it out + + mozbuild = target_dir / "moz.build" + print(" ", " ", f"Writing {mozbuild}") + data = b"\n".join((x.encode() for x in lines)) + mozbuild.write_bytes(data) + + return required_files + + +# - + +for target_name in libraries: + reqs = export_target(target_name) + required_files |= reqs + +# Copy all the files + +print("\nMigrate required files") + +i = 0 +for x in required_files: + i += 1 + sys.stdout.write(f"\r Copying {i}/{len(required_files)}") + sys.stdout.flush() + assert x.startswith("//"), x + x = x[2:] + + src = REPO_DIR / x + dest = checkout_dir / x + + dest.parent.mkdir(parents=True, exist_ok=True) + data = src.read_bytes() + data = data.replace(b"\r\n", b"\n") + dest.write_bytes(data) + +print("\n\nDone") |