firefox/tools/use-moz-src/mach_commands.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import asyncio
import logging
import os
import re
from shutil import which

from mach.decorators import (
    Command,
    CommandArgument,
)

_log = logging.getLogger(__name__)

NO_RIPGREP_MSG = """
Could not find ripgrep. Perhaps it needs installing via your package manager
 or `cargo install`.
""".strip()

RIPGREP_ERROR_MSG = """
An error occurred running ripgrep. Please check the following error messages:

{}
""".strip()

NO_MODULES_TO_REWRITE_MSG = """
Found no EXTRA_JS_MODULES we could convert in the moz.build file(s) passed.
""".strip()

CANNOT_CONVERT_ERROR_MSG = """
Unfortunately {} cannot be automatically converted.

The script isn't clever enough to convert everything else while leaving that
module alone. Consider (temporarily) removing this module from the moz.build
file, then running the script, and then re-adding the module into the
moz.build file in a copy of the original EXTRA_JS_MODULES instruction.

The list of excluded module patterns is in tools/use-moz-src/mach_commands.py
and should have a comment explaining why it cannot be automatically moved to
moz-src:///.
""".strip()

excluded_from_convert_re = list(
    map(
        re.compile,
        [
            "BackgroundTask_.*\\.sys\\.mjs",  # Depend on this resource: path.
            "BrowserUsageTelemetry\\.sys\\.mjs",  # Browser + Android duplicates.
            "BuiltInThemes\\.sys\\.mjs",  # Thunderbird override
            "CrashManager\\.sys.\\.mjs",  # Preprocessed.
            "ExtensionBrowsingData\\.sys\\.mjs",  # Browser + Android duplicates.
            "policies/schema\\.sys\\.mjs",  # Thunderbird override
            "Policies\\.sys\\.mjs",  # Thunderbird override
            "PromiseWorker\\.m?js",  # Preprocessed.
            "RFPTargetConstants\\.sys\\.mjs",  # Preprocessed.
            "ThemeVariableMap\\.sys\\.mjs",  # Thunderbird override
        ],
    )
)


def is_excluded_from_convert(path):
    """Returns true if the JSM file shouldn't be converted to ESM."""
    path_str = str(path)
    for expr in excluded_from_convert_re:
        if expr.search(path_str):
            return True

    return False


def extract_info_from_mozbuild(command_context, paths):
    mozbuilds_for_fixing = set()
    urlmap = dict()
    reader = command_context.mozbuild_reader(config_mode="empty")
    for mozbuild_path in paths:
        is_browser = mozbuild_path.startswith("browser") or (
            mozbuild_path.startswith("devtools")
            and not mozbuild_path.startswith("devtools/platform")
        )
        assignments = reader.find_variables_from_ast(
            variables="EXTRA_JS_MODULES", path=mozbuild_path, all_relevant_files=False
        )
        for path, _variable, key, value in assignments:
            module_path = path.replace("moz.build", "") + value
            if is_excluded_from_convert(module_path):
                _log.log(logging.ERROR, CANNOT_CONVERT_ERROR_MSG.format(value))
                return [], dict()

            newurl = "moz-src:///" + module_path
            module_name = os.path.basename(module_path)
            keystr = "/".join(key.split(".")) + "/" if key else ""
            resource_suffix = "modules/" + keystr + module_name
            if is_browser:
                urlmap["resource:///" + resource_suffix] = newurl
            else:
                urlmap["resource://gre/" + resource_suffix] = newurl

            mozbuilds_for_fixing.add(mozbuild_path)

    return mozbuilds_for_fixing, urlmap


extra_js_modules_re = re.compile('EXTRA_JS_MODULES(["\\.\\w/\\[\\]]*) [+=]+')


def rewrite_mozbuilds(mozbuilds):
    for path in mozbuilds:
        with open(path, "r+", encoding="utf-8", newline="\n") as f:
            contents = f.read()
            contents = re.sub(extra_js_modules_re, "MOZ_SRC_FILES +=", contents)
            f.seek(0)
            f.write(contents)
            f.truncate()


def _get_re_for_urls(urlmap):
    old_urls = list(urlmap.keys())

    prefix = os.path.commonprefix(old_urls)
    prefixlen = len(prefix)
    old_urls_without_prefix = map(lambda u: u[prefixlen:], old_urls)

    escaped_prefix = re.escape(prefix)
    # Put the "app" option in if needed:
    escaped_prefix = re.sub("resource:///", "resource://(app)?/", escaped_prefix)
    suffixes = "|".join(map(re.escape, old_urls_without_prefix))
    return escaped_prefix + "(" + suffixes + ")"


def handle_matching_file(file, urlmap_re, replace_fn):
    with open(file, "r+", encoding="utf-8", newline="\n") as f:
        contents = f.read()
        f.seek(0)
        contents = re.sub(urlmap_re, replace_fn, contents)
        f.write(contents)
        f.truncate()


async def find_and_replace_refs(urlmap):
    urlmap_re = _get_re_for_urls(urlmap)

    encoding = "utf-8"

    cmd_args = [which("rg"), "-l", "-e", urlmap_re]

    proc = await asyncio.subprocess.create_subprocess_exec(
        *cmd_args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
    )

    pending_file_writes = set()
    affected_files = set()

    def replace_fn(matchobj):
        old_url = matchobj.group(0)
        # Replace with the mapped URL or with itself if we can't find it.
        return urlmap.get(old_url, old_url)

    async with asyncio.TaskGroup() as tg:

        async def handle_matching_files() -> None:
            assert proc.stdout is not None
            while matching_file := await proc.stdout.readline():
                matching_file = matching_file.decode(encoding, "replace")
                if not matching_file.endswith("\n"):
                    continue
                matching_file = matching_file.strip()
                if not matching_file:
                    continue
                _log.log(logging.INFO, f"Rewriting {matching_file}")
                affected_files.add(matching_file)
                task = tg.create_task(
                    asyncio.to_thread(
                        handle_matching_file, matching_file, urlmap_re, replace_fn
                    )
                )
                pending_file_writes.add(task)
                task.add_done_callback(pending_file_writes.discard)

        async def handle_errors() -> None:
            assert proc.stderr is not None
            while chunk := await proc.stderr.read(10000):
                _log.log(
                    logging.ERROR,
                    RIPGREP_ERROR_MSG.format(chunk.decode("utf-8", "replace")),
                )
                # We assume that if anything gets written to stderr by ripgrep,
                # it'll also exit shortly and we don't need to deal with that
                # separately by raising exceptions or whatever.

        await asyncio.gather(handle_matching_files(), handle_errors(), proc.wait())
    return affected_files


@Command(
    "use-moz-src",
    category="misc",
    description="Convert use of EXTRA_JS_MODULES to MOZ_SRC_FILES",
)
@CommandArgument(
    "paths",
    nargs="+",
    help="Path to the moz.build file(s) to convert.",
)
def use_moz_src(command_context, paths):
    """
    This command does two things:
    1. replace use of EXTRA_JS_MODULES in the moz.build file passed with
       MOZ_SRC_FILES.
    2. fix up consumers across the tree that rely on any files in
       EXTRA_JS_MODULES using `resource` URLs to use `moz-src` ones
       instead.

    Note that this only uses the moz.build file in question; it will not
    recurse into `DIRS`. If you want to convert subdirectories, use
    e.g. `find` to collect the moz.build files and pass all of them on
    the commandline.

    Example:

    ./mach use-moz-src browser/components/moz.build

    Recursive example:

    find browser/components -iname 'moz.build' | xargs ./mach use-moz-src

    """
    if not which("rg"):
        _log.log(logging.ERROR, NO_RIPGREP_MSG)
        return

    mozbuilds_for_fixing, urlmap = extract_info_from_mozbuild(command_context, paths)

    if not mozbuilds_for_fixing:
        _log.log(logging.INFO, NO_MODULES_TO_REWRITE_MSG)
        return

    _log.log(
        logging.INFO,
        "Searching the tree and updating all references to:\n  {}".format(
            "\n  ".join(urlmap.keys())
        ),
    )
    updated_files = asyncio.run(find_and_replace_refs(urlmap))

    _log.log(
        logging.INFO,
        "Updating build files:\n  {}".format("\n  ".join(mozbuilds_for_fixing)),
    )

    rewrite_mozbuilds(mozbuilds_for_fixing)

    if len(updated_files) > 0:
        _log.log(logging.INFO, f"Formatting {len(updated_files)} affected files...")
        command_context._mach_context.commands.dispatch(
            "lint",
            command_context._mach_context,
            linters=["eslint"],
            paths=updated_files,
            fix=True,
        )

    _log.log(
        logging.INFO, "Done. Make sure to test the result before submitting patches."
    )