summaryrefslogtreecommitdiffstats
path: root/gfx/angle/update-angle.py
diff options
context:
space:
mode:
Diffstat (limited to 'gfx/angle/update-angle.py')
-rwxr-xr-xgfx/angle/update-angle.py560
1 files changed, 560 insertions, 0 deletions
diff --git a/gfx/angle/update-angle.py b/gfx/angle/update-angle.py
new file mode 100755
index 0000000000..1e047d1fac
--- /dev/null
+++ b/gfx/angle/update-angle.py
@@ -0,0 +1,560 @@
+#! /usr/bin/env python3
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+assert __name__ == "__main__"
+
+"""
+To update ANGLE in Gecko, use Windows with git-bash, and setup depot_tools, python2, and
+python3. Because depot_tools expects `python` to be `python2` (shame!), python2 must come
+before python3 in your path.
+
+Upstream: https://chromium.googlesource.com/angle/angle
+
+Our repo: https://github.com/mozilla/angle
+It has branches like 'firefox-60' which is the branch we use for pulling into
+Gecko with this script.
+
+This script leaves a record of the merge-base and cherry-picks that we pull into
+Gecko. (gfx/angle/cherries.log)
+
+ANGLE<->Chrome version mappings are here: https://omahaproxy.appspot.com/
+An easy choice is to grab Chrome's Beta's ANGLE branch.
+
+## Usage
+
+Prepare your env:
+
+* If in `cmd`:
+ `export PATH="$PATH:/path/to/depot_tools"`
+* If in `powershell`:
+ `$env:Path += ";C:\path\to\depot_tools"`
+
+If this is a new repo, don't forget:
+
+~~~
+# In the angle repo:
+./scripts/bootstrap.py
+gclient sync
+~~~
+
+Update: (in the angle repo)
+
+~~~
+# In the angle repo:
+/path/to/gecko/gfx/angle/update-angle.py origin
+git push moz # Push the firefox-XX branch to github.com/mozilla/angle
+~~~~
+
+"""
+
+import json
+import os
+import pathlib
+import re
+import shutil
+import subprocess
+import sys
+from typing import * # mypy annotations
+
+REPO_DIR = pathlib.Path.cwd()
+GECKO_ANGLE_DIR = pathlib.Path(__file__).parent
+
+OUT_DIR = pathlib.Path("out")
+
+COMMON_HEADER = [
+ "# Generated by update-angle.py",
+ "",
+ 'include("../../moz.build.common")',
+]
+
+ROOTS = ["//:translator", "//:libEGL", "//:libGLESv2"]
+
+CHECK_ONLY = False
+args = sys.argv[1:]
+while True:
+ arg = args.pop(0)
+ if arg == "--check":
+ CHECK_ONLY = True
+ continue
+ args.insert(0, arg)
+ break
+
+GN_ENV = dict(os.environ)
+GN_ENV["DEPOT_TOOLS_WIN_TOOLCHAIN"] = "0"
+
+(GIT_REMOTE,) = args # Not always 'origin'!
+
+# ------------------------------------------------------------------------------
+
+
+def run_checked(*args, **kwargs):
+ print(" ", args)
+ sys.stdout.flush()
+ return subprocess.run(args, check=True, **kwargs)
+
+
+def sorted_items(x):
+ for k in sorted(x.keys()):
+ yield (k, x[k])
+
+
+def collapse_dotdots(path):
+ split = path.split("/")
+
+ ret = []
+ for x in split:
+ if x == ".." and ret:
+ ret.pop()
+ continue
+ ret.append(x)
+ continue
+
+ return "/".join(ret)
+
+
+def dag_traverse(root_keys: Sequence[str], pre_recurse_func: Callable[[str], list]):
+ visited_keys: Set[str] = set()
+
+ def recurse(key):
+ if key in visited_keys:
+ return
+ visited_keys.add(key)
+
+ t = pre_recurse_func(key)
+ try:
+ (next_keys, post_recurse_func) = t
+ except ValueError:
+ (next_keys,) = t
+ post_recurse_func = None
+
+ for x in next_keys:
+ recurse(x)
+
+ if post_recurse_func:
+ post_recurse_func(key)
+ return
+
+ for x in root_keys:
+ recurse(x)
+ return
+
+
+# ------------------------------------------------------------------------------
+
+print("Importing graph")
+
+# shutil.rmtree(str(OUT_DIR), True)
+OUT_DIR.mkdir(exist_ok=True)
+
+GN_ARGS = b"""
+# Build arguments go here.
+# See "gn args <out_dir> --list" for available build arguments.
+is_clang = true
+is_debug = false
+angle_build_all = false
+angle_enable_abseil = false
+angle_enable_apple_translator_workarounds = true
+angle_enable_essl = true
+angle_enable_gl = false
+angle_enable_gl_desktop_frontend = false
+angle_enable_glsl = true
+angle_enable_null = false
+angle_enable_share_context_lock = true
+angle_enable_vulkan = false
+angle_has_astc_encoder = false
+use_custom_libcxx = false
+"""[
+ 1:
+]
+args_gn_path = OUT_DIR / "args.gn"
+args_gn_path.write_bytes(GN_ARGS)
+
+try:
+ run_checked("gn", "gen", str(OUT_DIR), shell=True, env=GN_ENV)
+except subprocess.CalledProcessError:
+ sys.stderr.buffer.write(b"`gn` failed. Is depot_tools in your PATH?\n")
+ exit(1)
+
+p = run_checked(
+ "python3",
+ "scripts/export_targets.py",
+ str(OUT_DIR),
+ *ROOTS,
+ stdout=subprocess.PIPE,
+ shell=True,
+ env=GN_ENV,
+)
+
+# -
+
+print("\nProcessing graph")
+libraries = json.loads(p.stdout.decode())
+
+# -
+# HACKHACKHACK: Inject linux/mac sources instead of trying to merge graphs of different
+# platforms.
+# descs["//:angle_common"]["sources"] +=
+EXTRA_ANGLE_COMMON_SOURCES = [
+ "//src/common/system_utils_apple.cpp",
+ "//src/common/system_utils_linux.cpp",
+ "//src/common/system_utils_mac.cpp",
+ "//src/common/system_utils_posix.cpp",
+]
+
+angle_common = libraries["//:angle_common"]
+angle_common["sources"] += EXTRA_ANGLE_COMMON_SOURCES
+angle_common["sources"] = sorted(angle_common["sources"])
+
+# -
+# Reuse our own zlib
+
+del libraries["//third_party/zlib:zlib"]
+
+# -
+
+if CHECK_ONLY:
+ print("\n--check complete.")
+ exit(0)
+
+# ------------------------------------------------------------------------------
+# Output to moz.builds
+
+import vendor_from_git
+
+print("")
+vendor_from_git.record_cherry_picks(GECKO_ANGLE_DIR, GIT_REMOTE)
+
+# --
+
+
+def sortedi(x):
+ return sorted(x, key=str.lower)
+
+
+def append_arr(dest, name, vals, indent=0):
+ if not vals:
+ return
+
+ dest.append("{}{} += [".format(" " * 4 * indent, name))
+ for x in sortedi(vals):
+ dest.append('{}"{}",'.format(" " * 4 * (indent + 1), x))
+ dest.append("{}]".format(" " * 4 * indent))
+ dest.append("")
+ return
+
+
+REGISTERED_DEFINES = {
+ "ADLER32_SIMD_SSSE3": False,
+ "ANGLE_CAPTURE_ENABLED": True,
+ "ANGLE_DISABLE_POOL_ALLOC": True,
+ "ANGLE_EGL_LIBRARY_NAME": False,
+ "ANGLE_ENABLE_APPLE_WORKAROUNDS": True,
+ "ANGLE_ENABLE_D3D11": True,
+ "ANGLE_ENABLE_D3D11_COMPOSITOR_NATIVE_WINDOW": True,
+ "ANGLE_ENABLE_D3D9": True,
+ "ANGLE_ENABLE_DEBUG_ANNOTATIONS": True,
+ "ANGLE_ENABLE_NULL": False,
+ "ANGLE_ENABLE_OPENGL": False,
+ "ANGLE_ENABLE_OPENGL_NULL": False,
+ "ANGLE_ENABLE_ESSL": True,
+ "ANGLE_ENABLE_GLSL": True,
+ "ANGLE_ENABLE_HLSL": True,
+ "ANGLE_ENABLE_SHARE_CONTEXT_LOCK": True,
+ "ANGLE_GENERATE_SHADER_DEBUG_INFO": True,
+ "ANGLE_GLESV2_LIBRARY_NAME": True,
+ "ANGLE_HAS_ASTCENC": True,
+ "ANGLE_HAS_VULKAN_SYSTEM_INFO": False,
+ "ANGLE_IS_64_BIT_CPU": False,
+ "ANGLE_IS_WIN": False,
+ "ANGLE_PRELOADED_D3DCOMPILER_MODULE_NAMES": False,
+ "ANGLE_SHARED_LIBVULKAN": True,
+ "ANGLE_USE_CUSTOM_LIBVULKAN": True,
+ "ANGLE_USE_EGL_LOADER": True,
+ "ANGLE_VK_LAYERS_DIR": True,
+ "ANGLE_VK_MOCK_ICD_JSON": True,
+ "ANGLE_VMA_VERSION": True,
+ "ASTCENC_DECOMPRESS_ONLY": True,
+ "CERT_CHAIN_PARA_HAS_EXTRA_FIELDS": False,
+ "CHROMIUM_BUILD": False,
+ "COMPONENT_BUILD": False,
+ "CRC32_SIMD_SSE42_PCLMUL": False,
+ "DEFLATE_FILL_WINDOW_SSE2": False,
+ "DYNAMIC_ANNOTATIONS_ENABLED": True,
+ "EGL_EGL_PROTOTYPES": True,
+ "EGL_EGLEXT_PROTOTYPES": True,
+ "EGLAPI": True,
+ "FIELDTRIAL_TESTING_ENABLED": False,
+ "FULL_SAFE_BROWSING": False,
+ "GL_API": True,
+ "GL_APICALL": True,
+ "GL_GLES_PROTOTYPES": True,
+ "GL_GLEXT_PROTOTYPES": True,
+ "GPU_INFO_USE_SETUPAPI": True,
+ "INFLATE_CHUNK_READ_64LE": False,
+ "INFLATE_CHUNK_SIMD_SSE2": False,
+ "LIBANGLE_IMPLEMENTATION": True,
+ "LIBEGL_IMPLEMENTATION": True,
+ "LIBGLESV2_IMPLEMENTATION": True,
+ "NOMINMAX": True,
+ "NO_TCMALLOC": False,
+ # Else: gfx/angle/checkout/src/libANGLE/renderer/d3d/d3d11/win32/NativeWindow11Win32.cpp(89): error C2787: 'IDCompositionDevice': no GUID has been associated with this object
+ "NTDDI_VERSION": False,
+ "PSAPI_VERSION": False,
+ "SAFE_BROWSING_CSD": False,
+ "SAFE_BROWSING_DB_LOCAL": False,
+ "UNICODE": True,
+ "USE_AURA": False,
+ "V8_DEPRECATION_WARNINGS": False,
+ "VK_USE_PLATFORM_WIN32_KHR": False,
+ "WIN32": False,
+ "WIN32_LEAN_AND_MEAN": False,
+ "WINAPI_FAMILY": False,
+ "WINVER": True,
+ # Otherwise:
+ # gfx/angle/targets/libANGLE
+ # In file included from c:/dev/mozilla/gecko4/gfx/angle/checkout/src/libANGLE/renderer/d3d/d3d11/converged/CompositorNativeWindow11.cpp:10:
+ # In file included from c:/dev/mozilla/gecko4/gfx/angle/checkout/src\libANGLE/renderer/d3d/d3d11/converged/CompositorNativeWindow11.h:17:
+ # C:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\winrt\Windows.ui.composition.interop.h(103,20): error: unknown type name 'POINTER_INFO'
+ # _In_ const POINTER_INFO& pointerInfo
+ # ^
+ "WTF_USE_DYNAMIC_ANNOTATIONS": False,
+ "X86_WINDOWS": False,
+ "ZLIB_IMPLEMENTATION": True,
+ "_ATL_NO_OPENGL": True,
+ "_CRT_NONSTDC_NO_DEPRECATE": True,
+ "_CRT_NONSTDC_NO_WARNINGS": True,
+ "_CRT_RAND_S": True,
+ "_CRT_SECURE_NO_DEPRECATE": True,
+ "_DEBUG": False,
+ "_HAS_EXCEPTIONS": True,
+ "_HAS_ITERATOR_DEBUGGING": False,
+ "_SCL_SECURE_NO_DEPRECATE": True,
+ "_SECURE_ATL": True,
+ "_UNICODE": True,
+ "_USING_V110_SDK71_": False,
+ "_WIN32_WINNT": False,
+ "_WINDOWS": False,
+ "_WINSOCK_DEPRECATED_NO_WARNINGS": True,
+ "__STD_C": False,
+ # clang specific
+ "CR_CLANG_REVISION": True,
+ "NDEBUG": False,
+ "NVALGRIND": False,
+ "_HAS_NODISCARD": False,
+}
+
+# -
+
+print("\nRun actions")
+required_files: Set[str] = set()
+required_files.add("//LICENSE")
+
+run_checked("ninja", "-C", str(OUT_DIR), ":angle_commit_id", shell=True)
+required_files.add("//out/gen/angle/angle_commit.h")
+
+# -
+
+# Export our targets
+print("\nExport targets")
+
+# Clear our dest directories
+targets_dir = pathlib.Path(GECKO_ANGLE_DIR, "targets")
+checkout_dir = pathlib.Path(GECKO_ANGLE_DIR, "checkout")
+
+shutil.rmtree(targets_dir, True)
+shutil.rmtree(checkout_dir, True)
+targets_dir.mkdir(exist_ok=True)
+checkout_dir.mkdir(exist_ok=True)
+
+# -
+
+RE_TARGET_NAME = re.compile("//(.*):(.+)")
+
+
+def export_target(target_full_name) -> Set[str]:
+ # print(' ', target_full_name)
+ descs = libraries
+ desc = descs[target_name]
+ flat = desc
+
+ m = RE_TARGET_NAME.match(target_name)
+ assert m, target_name
+ name = m.group(2)
+
+ required_files: Set[str] = set(flat["sources"])
+
+ # Create our manifest lines
+ target_dir = targets_dir / name
+ target_dir.mkdir(exist_ok=True)
+
+ lines = list(COMMON_HEADER)
+ lines.append("")
+
+ for x in sorted(set(desc["defines"])):
+ try:
+ (k, v) = x.split("=", 1)
+ if v.startswith('"'):
+ v = f"'{v}'"
+ else:
+ v = f'"{v}"'
+ except ValueError:
+ (k, v) = (x, "True")
+
+ line = f'DEFINES["{k}"] = {v}'
+ try:
+ if REGISTERED_DEFINES[k] == False:
+ line = "# " + line
+ except KeyError:
+ print(f"[{name}] Unrecognized define: {k}")
+ line = "# Unrecognized: " + line
+ lines.append(line)
+ lines.append("")
+
+ cxxflags = set(desc["cflags"] + desc["cflags_cc"])
+
+ def fixup_paths(listt):
+ for x in set(listt):
+ assert x.startswith("//"), x
+ yield "../../checkout/" + x[2:]
+
+ sources_by_config: Dict[str, List[str]] = {}
+ extras: Dict[str, str] = dict()
+ for x in fixup_paths(flat["sources"]):
+ # print(' '*5, x)
+ (b, e) = x.rsplit(".", 1)
+ if e in ["h", "hpp", "y", "l", "inc", "inl"]:
+ continue
+ elif e in ["cpp", "cc", "c"]:
+ if b.endswith("_win") or b.endswith("_win32"):
+ config = 'CONFIG["OS_ARCH"] == "WINNT"'
+ elif b.endswith("_linux"):
+ # Include these on BSDs too.
+ config = 'CONFIG["OS_ARCH"] not in ("Darwin", "WINNT")'
+ elif b.endswith("_apple") or b.endswith("_mac"):
+ config = 'CONFIG["OS_ARCH"] == "Darwin"'
+ elif b.endswith("_posix"):
+ config = 'CONFIG["OS_ARCH"] != "WINNT"'
+ else:
+ config = "" # None can't compare against str.
+
+ sources_by_config.setdefault(config, []).append(x)
+ continue
+ elif e == "rc":
+ assert "RCFILE" not in extras, (target_name, extras["RCFILE"], x)
+ extras["RCFILE"] = f'"{x}"'
+ continue
+ elif e == "def":
+ assert "DEFFILE" not in extras, (target_name, extras["DEFFILE"], x)
+ extras["DEFFILE"] = f'"{x}"'
+ continue
+ else:
+ assert False, ("Unhandled ext:", x)
+
+ ldflags = set(desc["ldflags"])
+ DEF_PREFIX = "/DEF:"
+ for x in set(ldflags):
+ if x.startswith(DEF_PREFIX):
+ def_path = x[len(DEF_PREFIX) :]
+ required_files.add(def_path)
+ assert "DEFFILE" not in extras
+ ldflags.remove(x)
+
+ def_path = str(OUT_DIR) + "/" + def_path
+ def_path = "//" + collapse_dotdots(def_path)
+
+ def_rel_path = list(fixup_paths([def_path]))[0]
+ extras["DEFFILE"] = '"{}"'.format(def_rel_path)
+ elif x.startswith("/PDBSourcePath:"):
+ ldflags.remove(x)
+
+ os_libs = list(map(lambda x: x[: -len(".lib")], set(desc.get("libs", []))))
+
+ def append_arr_commented(dest, name, src):
+ lines = []
+ append_arr(lines, name, src)
+
+ def comment(x):
+ if x:
+ x = "# " + x
+ return x
+
+ lines = map(comment, lines)
+ dest += lines
+
+ append_arr(lines, "LOCAL_INCLUDES", fixup_paths(desc["include_dirs"]))
+ append_arr_commented(lines, "CXXFLAGS", cxxflags)
+
+ for (config, v) in sorted_items(sources_by_config):
+ indent = 0
+ if config:
+ lines.append("if {}:".format(config))
+ indent = 1
+ append_arr(lines, "SOURCES", v, indent=indent)
+
+ dep_libs: Set[str] = set()
+ for dep_full_name in set(flat["dep_libs"]):
+ assert dep_full_name.startswith("//"), dep_name
+ (_, dep_name) = dep_full_name.split(":")
+ dep_libs.add(dep_name)
+
+ dep_dirs = set(dep_libs)
+ dep_dirs.discard("zlib")
+
+ append_arr(lines, "USE_LIBS", dep_libs)
+ append_arr(lines, "DIRS", ["../" + x for x in dep_dirs])
+ append_arr(lines, "OS_LIBS", os_libs)
+ append_arr_commented(lines, "LDFLAGS", ldflags)
+
+ for (k, v) in sorted(extras.items()):
+ lines.append("{} = {}".format(k, v))
+
+ lib_type = desc["type"]
+ if lib_type == "shared_library":
+ lines.append(f'GeckoSharedLibrary("{name}", linkage=None)')
+ elif lib_type == "static_library":
+ lines.append(f'Library("{name}")')
+ else:
+ assert False, lib_type
+
+ lines.append("")
+ # EOF newline
+
+ # Write it out
+
+ mozbuild = target_dir / "moz.build"
+ print(" ", " ", f"Writing {mozbuild}")
+ data = b"\n".join((x.encode() for x in lines))
+ mozbuild.write_bytes(data)
+
+ return required_files
+
+
+# -
+
+for target_name in libraries:
+ reqs = export_target(target_name)
+ required_files |= reqs
+
+# Copy all the files
+
+print("\nMigrate required files")
+
+i = 0
+for x in required_files:
+ i += 1
+ sys.stdout.write(f"\r Copying {i}/{len(required_files)}")
+ sys.stdout.flush()
+ assert x.startswith("//"), x
+ x = x[2:]
+
+ src = REPO_DIR / x
+ dest = checkout_dir / x
+
+ dest.parent.mkdir(parents=True, exist_ok=True)
+ data = src.read_bytes()
+ data = data.replace(b"\r\n", b"\n")
+ dest.write_bytes(data)
+
+print("\n\nDone")