path: root/gfx/angle/update-angle.py

#! /usr/bin/env python3
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

assert __name__ == "__main__"

r"""
To update ANGLE in Gecko, use Windows with git-bash, and setup depot_tools, python2, and
python3. Because depot_tools expects `python` to be `python2` (shame!), python2 must come
before python3 in your path.

Upstream: https://chromium.googlesource.com/angle/angle

Our repo: https://github.com/mozilla/angle
It has branches like 'firefox-60' which is the branch we use for pulling into
Gecko with this script.

This script leaves a record of the merge-base and cherry-picks that we pull into
Gecko. (gfx/angle/cherries.log)

ANGLE<->Chrome version mappings are here: https://omahaproxy.appspot.com/
An easy choice is to grab Chrome's Beta's ANGLE branch.

## Usage

Prepare your env:

* If in `cmd`:
    `export PATH="$PATH:/path/to/depot_tools"`
* If in `powershell`:
    `$env:Path += ";C:\path\to\depot_tools"`

If this is a new repo, don't forget:

~~~
# In the angle repo:
./scripts/bootstrap.py
gclient sync
~~~

Update: (in the angle repo)

~~~
# In the angle repo:
/path/to/gecko/gfx/angle/update-angle.py origin
git push moz # Push the firefox-XX branch to github.com/mozilla/angle
~~~~

"""

import json
import os
import pathlib
import re
import shutil
import subprocess
import sys
from typing import *  # mypy annotations

REPO_DIR = pathlib.Path.cwd()
GECKO_ANGLE_DIR = pathlib.Path(__file__).parent

OUT_DIR = pathlib.Path("out")

COMMON_HEADER = [
    "# Generated by update-angle.py",
    "",
    'include("../../moz.build.common")',
]

ROOTS = ["//:translator", "//:libEGL", "//:libGLESv2"]

CHECK_ONLY = False
args = sys.argv[1:]
while True:
    arg = args.pop(0)
    if arg == "--check":
        CHECK_ONLY = True
        continue
    args.insert(0, arg)
    break

GN_ENV = dict(os.environ)
GN_ENV["DEPOT_TOOLS_WIN_TOOLCHAIN"] = "0"

(GIT_REMOTE,) = args  # Not always 'origin'!

# ------------------------------------------------------------------------------


def run_checked(*args, **kwargs):
    print(" ", args)
    sys.stdout.flush()
    return subprocess.run(args, check=True, **kwargs)


def sorted_items(x):
    for k in sorted(x.keys()):
        yield (k, x[k])


def collapse_dotdots(path):
    split = path.split("/")

    ret = []
    for x in split:
        if x == ".." and ret:
            ret.pop()
            continue
        ret.append(x)
        continue

    return "/".join(ret)


def dag_traverse(root_keys: Sequence[str], pre_recurse_func: Callable[[str], list]):
    visited_keys: Set[str] = set()

    def recurse(key):
        if key in visited_keys:
            return
        visited_keys.add(key)

        t = pre_recurse_func(key)
        try:
            (next_keys, post_recurse_func) = t
        except ValueError:
            (next_keys,) = t
            post_recurse_func = None

        for x in next_keys:
            recurse(x)

        if post_recurse_func:
            post_recurse_func(key)
        return

    for x in root_keys:
        recurse(x)
    return


# ------------------------------------------------------------------------------

print("Importing graph")

# shutil.rmtree(str(OUT_DIR), True)
OUT_DIR.mkdir(exist_ok=True)

GN_ARGS = b"""
# Build arguments go here.
# See "gn args <out_dir> --list" for available build arguments.
is_clang = true
is_debug = false
angle_build_all = false
angle_enable_abseil = false
angle_enable_apple_translator_workarounds = true
angle_enable_essl = true
angle_enable_gl = false
angle_enable_gl_desktop_frontend = false
angle_enable_glsl = true
angle_enable_null = false
angle_enable_share_context_lock = true
angle_enable_vulkan = false
angle_has_astc_encoder = false
use_custom_libcxx = false
"""[
    1:
]
args_gn_path = OUT_DIR / "args.gn"
args_gn_path.write_bytes(GN_ARGS)

try:
    run_checked("gn", "gen", str(OUT_DIR), shell=True, env=GN_ENV)
except subprocess.CalledProcessError:
    sys.stderr.buffer.write(b"`gn` failed. Is depot_tools in your PATH?\n")
    exit(1)

p = run_checked(
    "python3",
    "scripts/export_targets.py",
    str(OUT_DIR),
    *ROOTS,
    stdout=subprocess.PIPE,
    shell=True,
    env=GN_ENV,
)

# -

print("\nProcessing graph")
libraries = json.loads(p.stdout.decode())

# -
# HACKHACKHACK: Inject linux/mac sources instead of trying to merge graphs of different
# platforms.
# descs["//:angle_common"]["sources"] +=
EXTRA_ANGLE_COMMON_SOURCES = [
    "//src/common/system_utils_apple.cpp",
    "//src/common/system_utils_linux.cpp",
    "//src/common/system_utils_mac.cpp",
    "//src/common/system_utils_posix.cpp",
]

angle_common = libraries["//:angle_common"]
angle_common["sources"] += EXTRA_ANGLE_COMMON_SOURCES
angle_common["sources"] = sorted(angle_common["sources"])

# -
# Reuse our own zlib

del libraries["//third_party/zlib:zlib"]

# -

if CHECK_ONLY:
    print("\n--check complete.")
    exit(0)

# ------------------------------------------------------------------------------
# Output to moz.builds

import vendor_from_git

print("")
vendor_from_git.record_cherry_picks(GECKO_ANGLE_DIR, GIT_REMOTE)

# --


def sortedi(x):
    return sorted(x, key=str.lower)


def append_arr(dest, name, vals, indent=0):
    if not vals:
        return

    dest.append("{}{} += [".format(" " * 4 * indent, name))
    for x in sortedi(vals):
        dest.append('{}"{}",'.format(" " * 4 * (indent + 1), x))
    dest.append("{}]".format(" " * 4 * indent))
    dest.append("")
    return


REGISTERED_DEFINES = {
    "ADLER32_SIMD_SSSE3": False,
    "ANGLE_CAPTURE_ENABLED": True,
    "ANGLE_DISABLE_POOL_ALLOC": True,
    "ANGLE_EGL_LIBRARY_NAME": False,
    "ANGLE_ENABLE_APPLE_WORKAROUNDS": True,
    "ANGLE_ENABLE_D3D11": True,
    "ANGLE_ENABLE_D3D11_COMPOSITOR_NATIVE_WINDOW": True,
    "ANGLE_ENABLE_D3D9": True,
    "ANGLE_ENABLE_DEBUG_ANNOTATIONS": True,
    "ANGLE_ENABLE_NULL": False,
    "ANGLE_ENABLE_OPENGL": False,
    "ANGLE_ENABLE_OPENGL_NULL": False,
    "ANGLE_ENABLE_ESSL": True,
    "ANGLE_ENABLE_GLSL": True,
    "ANGLE_ENABLE_HLSL": True,
    "ANGLE_ENABLE_SHARE_CONTEXT_LOCK": True,
    "ANGLE_GENERATE_SHADER_DEBUG_INFO": True,
    "ANGLE_GLESV2_LIBRARY_NAME": True,
    "ANGLE_HAS_ASTCENC": True,
    "ANGLE_HAS_VULKAN_SYSTEM_INFO": False,
    "ANGLE_IS_64_BIT_CPU": False,
    "ANGLE_IS_WIN": False,
    "ANGLE_PRELOADED_D3DCOMPILER_MODULE_NAMES": False,
    "ANGLE_SHARED_LIBVULKAN": True,
    "ANGLE_USE_CUSTOM_LIBVULKAN": True,
    "ANGLE_USE_EGL_LOADER": True,
    "ANGLE_VK_LAYERS_DIR": True,
    "ANGLE_VK_MOCK_ICD_JSON": True,
    "ANGLE_VMA_VERSION": True,
    "ASTCENC_DECOMPRESS_ONLY": True,
    "CERT_CHAIN_PARA_HAS_EXTRA_FIELDS": False,
    "CHROMIUM_BUILD": False,
    "COMPONENT_BUILD": False,
    "CRC32_SIMD_SSE42_PCLMUL": False,
    "DEFLATE_FILL_WINDOW_SSE2": False,
    "DYNAMIC_ANNOTATIONS_ENABLED": True,
    "EGL_EGL_PROTOTYPES": True,
    "EGL_EGLEXT_PROTOTYPES": True,
    "EGLAPI": True,
    "FIELDTRIAL_TESTING_ENABLED": False,
    "FULL_SAFE_BROWSING": False,
    "GL_API": True,
    "GL_APICALL": True,
    "GL_GLES_PROTOTYPES": True,
    "GL_GLEXT_PROTOTYPES": True,
    "GPU_INFO_USE_SETUPAPI": True,
    "INFLATE_CHUNK_READ_64LE": False,
    "INFLATE_CHUNK_SIMD_SSE2": False,
    "LIBANGLE_IMPLEMENTATION": True,
    "LIBEGL_IMPLEMENTATION": True,
    "LIBGLESV2_IMPLEMENTATION": True,
    "NOMINMAX": True,
    "NO_TCMALLOC": False,
    # Else: gfx/angle/checkout/src/libANGLE/renderer/d3d/d3d11/win32/NativeWindow11Win32.cpp(89): error C2787: 'IDCompositionDevice': no GUID has been associated with this object
    "NTDDI_VERSION": False,
    "PSAPI_VERSION": False,
    "SAFE_BROWSING_CSD": False,
    "SAFE_BROWSING_DB_LOCAL": False,
    "UNICODE": True,
    "USE_AURA": False,
    "V8_DEPRECATION_WARNINGS": False,
    "VK_USE_PLATFORM_WIN32_KHR": False,
    "WIN32": False,
    "WIN32_LEAN_AND_MEAN": False,
    "WINAPI_FAMILY": False,
    "WINVER": True,
    # Otherwise:
    # gfx/angle/targets/libANGLE
    # In file included from c:/dev/mozilla/gecko4/gfx/angle/checkout/src/libANGLE/renderer/d3d/d3d11/converged/CompositorNativeWindow11.cpp:10:
    # In file included from c:/dev/mozilla/gecko4/gfx/angle/checkout/src\libANGLE/renderer/d3d/d3d11/converged/CompositorNativeWindow11.h:17:
    # C:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\winrt\Windows.ui.composition.interop.h(103,20): error: unknown type name 'POINTER_INFO'
    #         _In_ const POINTER_INFO& pointerInfo
    #                    ^
    "WTF_USE_DYNAMIC_ANNOTATIONS": False,
    "X86_WINDOWS": False,
    "ZLIB_IMPLEMENTATION": True,
    "_ATL_NO_OPENGL": True,
    "_CRT_NONSTDC_NO_DEPRECATE": True,
    "_CRT_NONSTDC_NO_WARNINGS": True,
    "_CRT_RAND_S": True,
    "_CRT_SECURE_NO_DEPRECATE": True,
    "_DEBUG": False,
    "_HAS_EXCEPTIONS": True,
    "_HAS_ITERATOR_DEBUGGING": False,
    "_SCL_SECURE_NO_DEPRECATE": True,
    "_SECURE_ATL": True,
    "_UNICODE": True,
    "_USING_V110_SDK71_": False,
    "_WIN32_WINNT": False,
    "_WINDOWS": False,
    "_WINSOCK_DEPRECATED_NO_WARNINGS": True,
    "__STD_C": False,
    # clang specific
    "CR_CLANG_REVISION": True,
    "NDEBUG": False,
    "NVALGRIND": False,
    "_HAS_NODISCARD": False,
}

# -

print("\nRun actions")
required_files: Set[str] = set()
required_files.add("//LICENSE")

run_checked("ninja", "-C", str(OUT_DIR), ":angle_commit_id", shell=True)
required_files.add("//out/gen/angle/angle_commit.h")

# -

# Export our targets
print("\nExport targets")

# Clear our dest directories
targets_dir = pathlib.Path(GECKO_ANGLE_DIR, "targets")
checkout_dir = pathlib.Path(GECKO_ANGLE_DIR, "checkout")

shutil.rmtree(targets_dir, True)
shutil.rmtree(checkout_dir, True)
targets_dir.mkdir(exist_ok=True)
checkout_dir.mkdir(exist_ok=True)

# -

RE_TARGET_NAME = re.compile("//(.*):(.+)")


def export_target(target_full_name) -> Set[str]:
    # print(' ', target_full_name)
    descs = libraries
    desc = descs[target_name]
    flat = desc

    m = RE_TARGET_NAME.match(target_name)
    assert m, target_name
    name = m.group(2)

    required_files: Set[str] = set(flat["sources"])

    # Create our manifest lines
    target_dir = targets_dir / name
    target_dir.mkdir(exist_ok=True)

    lines = list(COMMON_HEADER)
    lines.append("")

    for x in sorted(set(desc["defines"])):
        try:
            (k, v) = x.split("=", 1)
            if v.startswith('"'):
                v = f"'{v}'"
            else:
                v = f'"{v}"'
        except ValueError:
            (k, v) = (x, "True")

        line = f'DEFINES["{k}"] = {v}'
        try:
            if REGISTERED_DEFINES[k] == False:
                line = "# " + line
        except KeyError:
            print(f"[{name}] Unrecognized define: {k}")
            line = "# Unrecognized: " + line
        lines.append(line)
    lines.append("")

    cxxflags = set(desc["cflags"] + desc["cflags_cc"])

    def fixup_paths(listt):
        for x in set(listt):
            assert x.startswith("//"), x
            yield "../../checkout/" + x[2:]

    sources_by_config: Dict[str, List[str]] = {}
    extras: Dict[str, str] = dict()
    for x in fixup_paths(flat["sources"]):
        # print(' '*5, x)
        (b, e) = x.rsplit(".", 1)
        if e in ["h", "hpp", "y", "l", "inc", "inl"]:
            continue
        elif e in ["cpp", "cc", "c"]:
            if b.endswith("_win") or b.endswith("_win32"):
                config = 'CONFIG["OS_ARCH"] == "WINNT"'
            elif b.endswith("_linux"):
                # Include these on BSDs too.
                config = 'CONFIG["OS_ARCH"] not in ("Darwin", "WINNT")'
            elif b.endswith("_apple") or b.endswith("_mac"):
                config = 'CONFIG["OS_ARCH"] == "Darwin"'
            elif b.endswith("_posix"):
                config = 'CONFIG["OS_ARCH"] != "WINNT"'
            else:
                config = ""  # None can't compare against str.

            sources_by_config.setdefault(config, []).append(x)
            continue
        elif e == "rc":
            assert "RCFILE" not in extras, (target_name, extras["RCFILE"], x)
            extras["RCFILE"] = f'"{x}"'
            continue
        elif e == "def":
            assert "DEFFILE" not in extras, (target_name, extras["DEFFILE"], x)
            extras["DEFFILE"] = f'"{x}"'
            continue
        else:
            assert False, ("Unhandled ext:", x)

    ldflags = set(desc["ldflags"])
    DEF_PREFIX = "/DEF:"
    for x in set(ldflags):
        if x.startswith(DEF_PREFIX):
            def_path = x[len(DEF_PREFIX) :]
            required_files.add(def_path)
            assert "DEFFILE" not in extras
            ldflags.remove(x)

            def_path = str(OUT_DIR) + "/" + def_path
            def_path = "//" + collapse_dotdots(def_path)

            def_rel_path = list(fixup_paths([def_path]))[0]
            extras["DEFFILE"] = '"{}"'.format(def_rel_path)
        elif x.startswith("/PDBSourcePath:"):
            ldflags.remove(x)

    os_libs = list(map(lambda x: x[: -len(".lib")], set(desc.get("libs", []))))

    def append_arr_commented(dest, name, src):
        lines = []
        append_arr(lines, name, src)

        def comment(x):
            if x:
                x = "# " + x
            return x

        lines = map(comment, lines)
        dest += lines

    append_arr(lines, "LOCAL_INCLUDES", fixup_paths(desc["include_dirs"]))
    append_arr_commented(lines, "CXXFLAGS", cxxflags)

    for config, v in sorted_items(sources_by_config):
        indent = 0
        if config:
            lines.append("if {}:".format(config))
            indent = 1
        append_arr(lines, "SOURCES", v, indent=indent)

    dep_libs: Set[str] = set()
    for dep_full_name in set(flat["dep_libs"]):
        assert dep_full_name.startswith("//"), dep_name
        (_, dep_name) = dep_full_name.split(":")
        dep_libs.add(dep_name)

    dep_dirs = set(dep_libs)
    dep_dirs.discard("zlib")

    # Those directories are added by gfx/angle/moz.build.
    already_added_dirs = [
        "angle_common",
        "translator",
        "libEGL",
        "libGLESv2",
    ]

    append_arr(lines, "USE_LIBS", dep_libs)
    append_arr(
        lines, "DIRS", ["../" + x for x in dep_dirs if x not in already_added_dirs]
    )
    append_arr(lines, "OS_LIBS", os_libs)
    append_arr_commented(lines, "LDFLAGS", ldflags)

    for k, v in sorted(extras.items()):
        lines.append("{} = {}".format(k, v))

    lib_type = desc["type"]
    if lib_type == "shared_library":
        lines.append(f'GeckoSharedLibrary("{name}", linkage=None)')
    elif lib_type == "static_library":
        lines.append(f'Library("{name}")')
    else:
        assert False, lib_type

    lines.append("")
    # EOF newline

    # Write it out

    mozbuild = target_dir / "moz.build"
    print(" ", " ", f"Writing {mozbuild}")
    data = b"\n".join((x.encode() for x in lines))
    mozbuild.write_bytes(data)

    return required_files


# -

for target_name in libraries:
    reqs = export_target(target_name)
    required_files |= reqs

# Copy all the files

print("\nMigrate required files")

i = 0
for x in required_files:
    i += 1
    sys.stdout.write(f"\r  Copying {i}/{len(required_files)}")
    sys.stdout.flush()
    assert x.startswith("//"), x
    x = x[2:]

    src = REPO_DIR / x
    dest = checkout_dir / x

    dest.parent.mkdir(parents=True, exist_ok=True)
    data = src.read_bytes()
    data = data.replace(b"\r\n", b"\n")
    dest.write_bytes(data)

print("\n\nDone")