summaryrefslogtreecommitdiffstats
path: root/tools/esmify/mach_commands.py
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--tools/esmify/mach_commands.py910
1 files changed, 910 insertions, 0 deletions
diff --git a/tools/esmify/mach_commands.py b/tools/esmify/mach_commands.py
new file mode 100644
index 0000000000..787de48424
--- /dev/null
+++ b/tools/esmify/mach_commands.py
@@ -0,0 +1,910 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import logging
+import os
+import pathlib
+import re
+import subprocess
+import sys
+
+from mach.decorators import Command, CommandArgument
+
+
+def path_sep_to_native(path_str):
+ """Make separators in the path OS native."""
+ return pathlib.os.sep.join(path_str.split("/"))
+
+
+def path_sep_from_native(path):
+ """Make separators in the path OS native."""
+ return "/".join(str(path).split(pathlib.os.sep))
+
+
+excluded_from_convert_prefix = list(
+ map(
+ path_sep_to_native,
+ [
+ # Testcases for actors.
+ "toolkit/actors/TestProcessActorChild.jsm",
+ "toolkit/actors/TestProcessActorParent.jsm",
+ "toolkit/actors/TestWindowChild.jsm",
+ "toolkit/actors/TestWindowParent.jsm",
+ "js/xpconnect/tests/unit/",
+ # Testcase for build system.
+ "python/mozbuild/mozbuild/test/",
+ ],
+ )
+)
+
+
+def is_excluded_from_convert(path):
+ """Returns true if the JSM file shouldn't be converted to ESM."""
+ path_str = str(path)
+ for prefix in excluded_from_convert_prefix:
+ if path_str.startswith(prefix):
+ return True
+
+ return False
+
+
+excluded_from_imports_prefix = list(
+ map(
+ path_sep_to_native,
+ [
+ # Vendored or auto-generated files.
+ "browser/components/pocket/content/panels/js/vendor.bundle.js",
+ "devtools/client/debugger/dist/parser-worker.js",
+ "devtools/client/debugger/test/mochitest/examples/react/build/main.js",
+ "devtools/client/debugger/test/mochitest/examples/sourcemapped/polyfill-bundle.js",
+ "devtools/client/inspector/markup/test/shadowdom_open_debugger.min.js",
+ "devtools/client/shared/source-map-loader/test/browser/fixtures/bundle.js",
+ "layout/style/test/property_database.js",
+ "services/fxaccounts/FxAccountsPairingChannel.js",
+ "testing/talos/talos/tests/devtools/addon/content/pages/custom/debugger/static/js/main.js", # noqa E501
+ "testing/web-platform/",
+ # Unrelated testcases that has edge case syntax.
+ "browser/components/sessionstore/test/unit/data/",
+ "devtools/client/debugger/src/workers/parser/tests/fixtures/",
+ "devtools/client/debugger/test/mochitest/examples/sourcemapped/fixtures/",
+ "devtools/client/webconsole/test/browser/test-syntaxerror-worklet.js",
+ "devtools/server/tests/xpcshell/test_framebindings-03.js",
+ "devtools/server/tests/xpcshell/test_framebindings-04.js",
+ "devtools/shared/tests/xpcshell/test_eventemitter_basic.js",
+ "devtools/shared/tests/xpcshell/test_eventemitter_static.js",
+ "dom/base/crashtests/module-with-syntax-error.js",
+ "dom/base/test/file_bug687859-16.js",
+ "dom/base/test/file_bug687859-16.js",
+ "dom/base/test/file_js_cache_syntax_error.js",
+ "dom/base/test/jsmodules/module_badSyntax.js",
+ "dom/canvas/test/reftest/webgl-utils.js",
+ "dom/encoding/test/file_utf16_be_bom.js",
+ "dom/encoding/test/file_utf16_le_bom.js",
+ "dom/html/test/bug649134/file_bug649134-1.sjs",
+ "dom/html/test/bug649134/file_bug649134-2.sjs",
+ "dom/media/webrtc/tests/mochitests/identity/idp-bad.js",
+ "dom/serviceworkers/test/file_js_cache_syntax_error.js",
+ "dom/serviceworkers/test/parse_error_worker.js",
+ "dom/workers/test/importScripts_worker_imported3.js",
+ "dom/workers/test/invalid.js",
+ "dom/workers/test/threadErrors_worker1.js",
+ "dom/xhr/tests/browser_blobFromFile.js",
+ "image/test/browser/browser_image.js",
+ "js/xpconnect/tests/chrome/test_bug732665_meta.js",
+ "js/xpconnect/tests/mochitest/class_static_worker.js",
+ "js/xpconnect/tests/unit/bug451678_subscript.js",
+ "js/xpconnect/tests/unit/error_other.sys.mjs",
+ "js/xpconnect/tests/unit/es6module_parse_error.js",
+ "js/xpconnect/tests/unit/recursive_importA.jsm",
+ "js/xpconnect/tests/unit/recursive_importB.jsm",
+ "js/xpconnect/tests/unit/syntax_error.jsm",
+ "js/xpconnect/tests/unit/test_defineModuleGetter.js",
+ "js/xpconnect/tests/unit/test_import.js",
+ "js/xpconnect/tests/unit/test_import_shim.js",
+ "js/xpconnect/tests/unit/test_recursive_import.js",
+ "js/xpconnect/tests/unit/test_unload.js",
+ "modules/libpref/test/unit/data/testParser.js",
+ "python/mozbuild/mozbuild/test/",
+ "remote/shared/messagehandler/test/browser/resources/modules/root/invalid.sys.mjs",
+ "testing/talos/talos/startup_test/sessionrestore/profile-manywindows/sessionstore.js",
+ "testing/talos/talos/startup_test/sessionrestore/profile/sessionstore.js",
+ "toolkit/components/reader/Readerable.sys.mjs",
+ "toolkit/components/workerloader/tests/moduleF-syntax-error.js",
+ "tools/lint/test/",
+ "tools/update-packaging/test/",
+ # SpiderMonkey internals.
+ "js/examples/",
+ "js/src/",
+ # Files has macro.
+ "browser/app/profile/firefox.js",
+ "browser/branding/official/pref/firefox-branding.js",
+ "browser/components/enterprisepolicies/schemas/schema.sys.mjs",
+ "browser/locales/en-US/firefox-l10n.js",
+ "mobile/android/app/geckoview-prefs.js",
+ "mobile/android/app/mobile.js",
+ "mobile/android/locales/en-US/mobile-l10n.js",
+ "modules/libpref/greprefs.js",
+ "modules/libpref/init/all.js",
+ "testing/condprofile/condprof/tests/profile/user.js",
+ "testing/mozbase/mozprofile/tests/files/prefs_with_comments.js",
+ "toolkit/modules/AppConstants.sys.mjs",
+ "toolkit/mozapps/update/tests/data/xpcshellConstantsPP.js",
+ # Uniffi templates
+ "toolkit/components/uniffi-bindgen-gecko-js/src/templates/js/",
+ ],
+ )
+)
+
+EXCLUSION_FILES = [
+ os.path.join("tools", "rewriting", "Generated.txt"),
+ os.path.join("tools", "rewriting", "ThirdPartyPaths.txt"),
+]
+
+
+def load_exclusion_files():
+ for path in EXCLUSION_FILES:
+ with open(path, "r") as f:
+ for line in f:
+ p = path_sep_to_native(re.sub("\*$", "", line.strip()))
+ excluded_from_imports_prefix.append(p)
+
+
+def is_excluded_from_imports(path):
+ """Returns true if the JS file content shouldn't be handled by
+ jscodeshift.
+
+ This filter is necessary because jscodeshift cannot handle some
+ syntax edge cases and results in unexpected rewrite."""
+ path_str = str(path)
+ for prefix in excluded_from_imports_prefix:
+ if path_str.startswith(prefix):
+ return True
+
+ return False
+
+
+# Wrapper for hg/git operations
+class VCSUtils:
+ def run(self, cmd):
+ # Do not pass check=True because the pattern can match no file.
+ lines = subprocess.run(cmd, stdout=subprocess.PIPE).stdout.decode()
+ return filter(lambda x: x != "", lines.split("\n"))
+
+
+class HgUtils(VCSUtils):
+ def is_available():
+ return pathlib.Path(".hg").exists()
+
+ def rename(self, before, after):
+ cmd = ["hg", "rename", before, after]
+ subprocess.run(cmd, check=True)
+
+ def find_jsms(self, path):
+ jsms = []
+
+ # NOTE: `set:glob:` syntax does not accept backslash on windows.
+ path = path_sep_from_native(path)
+
+ cmd = ["hg", "files", f'set:glob:"{path}/**/*.jsm"']
+ for line in self.run(cmd):
+ jsm = pathlib.Path(line)
+ if is_excluded_from_convert(jsm):
+ continue
+ jsms.append(jsm)
+
+ cmd = [
+ "hg",
+ "files",
+ f"set:grep('EXPORTED_SYMBOLS = \[') and glob:\"{path}/**/*.js\"",
+ ]
+ for line in self.run(cmd):
+ jsm = pathlib.Path(line)
+ if is_excluded_from_convert(jsm):
+ continue
+ jsms.append(jsm)
+
+ return jsms
+
+ def find_all_jss(self, path):
+ jss = []
+
+ # NOTE: `set:glob:` syntax does not accept backslash on windows.
+ path = path_sep_from_native(path)
+
+ cmd = [
+ "hg",
+ "files",
+ f'set:glob:"{path}/**/*.jsm" or glob:"{path}/**/*.js" or '
+ + f'glob:"{path}/**/*.mjs" or glob:"{path}/**/*.sjs"',
+ ]
+ for line in self.run(cmd):
+ js = pathlib.Path(line)
+ if is_excluded_from_imports(js):
+ continue
+ jss.append(js)
+
+ return jss
+
+
+class GitUtils(VCSUtils):
+ def is_available():
+ return pathlib.Path(".git").exists()
+
+ def rename(self, before, after):
+ cmd = ["git", "mv", before, after]
+ subprocess.run(cmd, check=True)
+
+ def find_jsms(self, path):
+ jsms = []
+
+ cmd = ["git", "ls-files", f"{path}/*.jsm"]
+ for line in self.run(cmd):
+ jsm = pathlib.Path(line)
+ if is_excluded_from_convert(jsm):
+ continue
+ jsms.append(jsm)
+
+ handled = {}
+ cmd = ["git", "grep", "EXPORTED_SYMBOLS = \[", f"{path}/*.js"]
+ for line in self.run(cmd):
+ m = re.search("^([^:]+):", line)
+ if not m:
+ continue
+ filename = m.group(1)
+ if filename in handled:
+ continue
+ handled[filename] = True
+ jsm = pathlib.Path(filename)
+ if is_excluded_from_convert(jsm):
+ continue
+ jsms.append(jsm)
+
+ return jsms
+
+ def find_all_jss(self, path):
+ jss = []
+
+ cmd = [
+ "git",
+ "ls-files",
+ f"{path}/*.jsm",
+ f"{path}/*.js",
+ f"{path}/*.mjs",
+ f"{path}/*.sjs",
+ ]
+ for line in self.run(cmd):
+ js = pathlib.Path(line)
+ if is_excluded_from_imports(js):
+ continue
+ jss.append(js)
+
+ return jss
+
+
+class Summary:
+ def __init__(self):
+ self.convert_errors = []
+ self.import_errors = []
+ self.rename_errors = []
+ self.no_refs = []
+
+
+@Command(
+ "esmify",
+ category="misc",
+ description="ESMify JSM files.",
+)
+@CommandArgument(
+ "path",
+ nargs=1,
+ help="Path to the JSM file to ESMify, or the directory that contains "
+ "JSM files and/or JS files that imports ESM-ified JSM.",
+)
+@CommandArgument(
+ "--convert",
+ action="store_true",
+ help="Only perform the step 1 = convert part",
+)
+@CommandArgument(
+ "--imports",
+ action="store_true",
+ help="Only perform the step 2 = import calls part",
+)
+@CommandArgument(
+ "--prefix",
+ default="",
+ help="Restrict the target of import in the step 2 to ESM-ified JSM, by the "
+ "prefix match for the JSM file's path. e.g. 'browser/'.",
+)
+def esmify(command_context, path=None, convert=False, imports=False, prefix=""):
+ """
+ This command does the following 2 steps:
+ 1. Convert the JSM file specified by `path` to ESM file, or the JSM files
+ inside the directory specified by `path` to ESM files, and also
+ fix references in build files and test definitions
+ 2. Convert import calls inside file(s) specified by `path` for ESM-ified
+ files to use new APIs
+
+ Example 1:
+ # Convert all JSM files inside `browser/components/pagedata` directory,
+ # and replace all references for ESM-ified files in the entire tree to use
+ # new APIs
+
+ $ ./mach esmify --convert browser/components/pagedata
+ $ ./mach esmify --imports . --prefix=browser/components/pagedata
+
+ Example 2:
+ # Convert all JSM files inside `browser` directory, and replace all
+ # references for the JSM files inside `browser` directory to use
+ # new APIs
+
+ $ ./mach esmify browser
+ """
+
+ def error(text):
+ command_context.log(logging.ERROR, "esmify", {}, f"[ERROR] {text}")
+
+ def warn(text):
+ command_context.log(logging.WARN, "esmify", {}, f"[WARN] {text}")
+
+ def info(text):
+ command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}")
+
+ # If no options is specified, perform both.
+ if not convert and not imports:
+ convert = True
+ imports = True
+
+ path = pathlib.Path(path[0])
+
+ if not verify_path(command_context, path):
+ return 1
+
+ if HgUtils.is_available():
+ vcs_utils = HgUtils()
+ elif GitUtils.is_available():
+ vcs_utils = GitUtils()
+ else:
+ error(
+ "This script needs to be run inside mozilla-central "
+ "checkout of either mercurial or git."
+ )
+ return 1
+
+ load_exclusion_files()
+
+ info("Setting up jscodeshift...")
+ setup_jscodeshift()
+
+ is_single_file = path.is_file()
+
+ modified_files = []
+ summary = Summary()
+
+ if convert:
+ info("Searching files to convert to ESM...")
+ if is_single_file:
+ jsms = [path]
+ else:
+ jsms = vcs_utils.find_jsms(path)
+
+ info(f"Found {len(jsms)} file(s) to convert to ESM.")
+
+ info("Converting to ESM...")
+ jsms = convert_module(jsms, summary)
+ if jsms is None:
+ error("Failed to rewrite exports.")
+ return 1
+
+ info("Renaming...")
+ esms = rename_jsms(command_context, vcs_utils, jsms, summary)
+
+ modified_files += esms
+
+ if imports:
+ info("Searching files to rewrite imports...")
+
+ if is_single_file:
+ if convert:
+ # Already converted above
+ jss = esms
+ else:
+ jss = [path]
+ else:
+ jss = vcs_utils.find_all_jss(path)
+
+ info(f"Checking {len(jss)} JS file(s). Rewriting any matching imports...")
+
+ result = rewrite_imports(jss, prefix, summary)
+ if result is None:
+ return 1
+
+ info(f"Rewritten {len(result)} file(s).")
+
+ # Only modified files needs eslint fix
+ modified_files += result
+
+ modified_files = list(set(modified_files))
+
+ info(f"Applying eslint --fix for {len(modified_files)} file(s)...")
+ eslint_fix(command_context, modified_files)
+
+ def print_files(f, errors):
+ for [path, message] in errors:
+ f(f" * {path}")
+ if message:
+ f(f" {message}")
+
+ if len(summary.convert_errors):
+ error("========")
+ error("Following files are not converted into ESM due to error:")
+ print_files(error, summary.convert_errors)
+
+ if len(summary.import_errors):
+ warn("========")
+ warn("Following files are not rewritten to import ESMs due to error:")
+ warn(
+ "(NOTE: Errors related to 'private names' are mostly due to "
+ " preprocessor macros in the file):"
+ )
+ print_files(warn, summary.import_errors)
+
+ if len(summary.rename_errors):
+ error("========")
+ error("Following files are not renamed due to error:")
+ print_files(error, summary.rename_errors)
+
+ if len(summary.no_refs):
+ warn("========")
+ warn("Following files are not found in any build files.")
+ warn("Please update references to those files manually:")
+ print_files(warn, summary.rename_errors)
+
+ return 0
+
+
+def verify_path(command_context, path):
+ """Check if the path passed to the command is valid relative path."""
+
+ def error(text):
+ command_context.log(logging.ERROR, "esmify", {}, f"[ERROR] {text}")
+
+ if not path.exists():
+ error(f"{path} does not exist.")
+ return False
+
+ if path.is_absolute():
+ error("Path must be a relative path from mozilla-central checkout.")
+ return False
+
+ return True
+
+
+def find_file(path, target):
+ """Find `target` file in ancestor of path."""
+ target_path = path.parent / target
+ if not target_path.exists():
+ if path.parent == path:
+ return None
+
+ return find_file(path.parent, target)
+
+ return target_path
+
+
+def try_rename_in(command_context, path, target, jsm_name, esm_name, jsm_path):
+ """Replace the occurrences of `jsm_name` with `esm_name` in `target`
+ file."""
+
+ def info(text):
+ command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}")
+
+ if type(target) is str:
+ # Target is specified by filename, that may exist somewhere in
+ # the jsm's directory or ancestor directories.
+ target_path = find_file(path, target)
+ if not target_path:
+ return False
+
+ # JSM should be specified with relative path in the file.
+ #
+ # Single moz.build or jar.mn can contain multiple files with same name.
+ # Search for relative path.
+ jsm_relative_path = jsm_path.relative_to(target_path.parent)
+ jsm_path_str = path_sep_from_native(str(jsm_relative_path))
+ else:
+ # Target is specified by full path.
+ target_path = target
+
+ # JSM should be specified with full path in the file.
+ jsm_path_str = path_sep_from_native(str(jsm_path))
+
+ jsm_path_re = re.compile(r"\b" + jsm_path_str.replace(".", r"\.") + r"\b")
+ jsm_name_re = re.compile(r"\b" + jsm_name.replace(".", r"\.") + r"\b")
+
+ modified = False
+ content = ""
+ with open(target_path, "r") as f:
+ for line in f:
+ if jsm_path_re.search(line):
+ modified = True
+ line = jsm_name_re.sub(esm_name, line)
+
+ content += line
+
+ if modified:
+ info(f" {str(target_path)}")
+ info(f" {jsm_name} => {esm_name}")
+ with open(target_path, "w", newline="\n") as f:
+ f.write(content)
+
+ return True
+
+
+def try_rename_uri_in(command_context, target, jsm_name, esm_name, jsm_uri, esm_uri):
+ """Replace the occurrences of `jsm_uri` with `esm_uri` in `target` file."""
+
+ def info(text):
+ command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}")
+
+ modified = False
+ content = ""
+ with open(target, "r") as f:
+ for line in f:
+ if jsm_uri in line:
+ modified = True
+ line = line.replace(jsm_uri, esm_uri)
+
+ content += line
+
+ if modified:
+ info(f" {str(target)}")
+ info(f" {jsm_name} => {esm_name}")
+ with open(target, "w", newline="\n") as f:
+ f.write(content)
+
+ return True
+
+
+def try_rename_components_conf(command_context, path, jsm_name, esm_name):
+ """Replace the occurrences of `jsm_name` with `esm_name` in components.conf
+ file."""
+
+ def info(text):
+ command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}")
+
+ target_path = find_file(path, "components.conf")
+ if not target_path:
+ return False
+
+ # Unlike try_rename_in, components.conf contains the URL instead of
+ # relative path, and also there are no known files with same name.
+ # Simply replace the filename.
+
+ with open(target_path, "r") as f:
+ content = f.read()
+
+ prop_re = re.compile(
+ "[\"']jsm[\"']:(.*)" + r"\b" + jsm_name.replace(".", r"\.") + r"\b"
+ )
+
+ if not prop_re.search(content):
+ return False
+
+ info(f" {str(target_path)}")
+ info(f" {jsm_name} => {esm_name}")
+
+ content = prop_re.sub(r"'esModule':\1" + esm_name, content)
+ with open(target_path, "w", newline="\n") as f:
+ f.write(content)
+
+ return True
+
+
+def esmify_name(name):
+ return re.sub(r"\.(jsm|js|jsm\.js)$", ".sys.mjs", name)
+
+
+def esmify_path(jsm_path):
+ jsm_name = jsm_path.name
+ esm_name = re.sub(r"\.(jsm|js|jsm\.js)$", ".sys.mjs", jsm_name)
+ esm_path = jsm_path.parent / esm_name
+ return esm_path
+
+
+path_to_uri_map = None
+
+
+def load_path_to_uri_map():
+ global path_to_uri_map
+
+ if path_to_uri_map:
+ return
+
+ if "ESMIFY_MAP_JSON" in os.environ:
+ json_map = pathlib.Path(os.environ["ESMIFY_MAP_JSON"])
+ else:
+ json_map = pathlib.Path(__file__).parent / "map.json"
+
+ with open(json_map, "r") as f:
+ uri_to_path_map = json.loads(f.read())
+
+ path_to_uri_map = dict()
+
+ for uri, paths in uri_to_path_map.items():
+ if type(paths) is str:
+ paths = [paths]
+
+ for path in paths:
+ path_to_uri_map[path] = uri
+
+
+def find_jsm_uri(jsm_path):
+ load_path_to_uri_map()
+
+ path = path_sep_from_native(jsm_path)
+
+ if path in path_to_uri_map:
+ return path_to_uri_map[path]
+
+ return None
+
+
+def rename_single_file(command_context, vcs_utils, jsm_path, summary):
+ """Rename `jsm_path` to .sys.mjs, and fix references to the file in build
+ and test definitions."""
+
+ def info(text):
+ command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}")
+
+ esm_path = esmify_path(jsm_path)
+
+ jsm_name = jsm_path.name
+ esm_name = esm_path.name
+
+ target_files = [
+ ".eslintignore",
+ "moz.build",
+ "jar.mn",
+ "browser.ini",
+ "browser-common.ini",
+ "chrome.ini",
+ "mochitest.ini",
+ "xpcshell.ini",
+ "xpcshell-child-process.ini",
+ "xpcshell-common.ini",
+ "xpcshell-parent-process.ini",
+ pathlib.Path("tools", "lint", "eslint.yml"),
+ pathlib.Path("tools", "lint", "rejected-words.yml"),
+ ]
+
+ info(f"{jsm_path} => {esm_path}")
+
+ renamed = False
+ for target in target_files:
+ if try_rename_in(
+ command_context, jsm_path, target, jsm_name, esm_name, jsm_path
+ ):
+ renamed = True
+
+ if try_rename_components_conf(command_context, jsm_path, jsm_name, esm_name):
+ renamed = True
+
+ uri_target_files = [
+ pathlib.Path(
+ "browser", "base", "content", "test", "performance", "browser_startup.js"
+ ),
+ pathlib.Path(
+ "browser",
+ "base",
+ "content",
+ "test",
+ "performance",
+ "browser_startup_content.js",
+ ),
+ pathlib.Path(
+ "browser",
+ "base",
+ "content",
+ "test",
+ "performance",
+ "browser_startup_content_subframe.js",
+ ),
+ pathlib.Path(
+ "toolkit",
+ "components",
+ "backgroundtasks",
+ "tests",
+ "browser",
+ "browser_xpcom_graph_wait.js",
+ ),
+ ]
+
+ jsm_uri = find_jsm_uri(jsm_path)
+ if jsm_uri:
+ esm_uri = re.sub(r"\.(jsm|js|jsm\.js)$", ".sys.mjs", jsm_uri)
+
+ for target in uri_target_files:
+ if try_rename_uri_in(
+ command_context, target, jsm_uri, esm_uri, jsm_name, esm_name
+ ):
+ renamed = True
+
+ if not renamed:
+ summary.no_refs.append([jsm_path, None])
+
+ if not esm_path.exists():
+ vcs_utils.rename(jsm_path, esm_path)
+ else:
+ summary.rename_errors.append([jsm_path, f"{esm_path} already exists"])
+
+ return esm_path
+
+
+def rename_jsms(command_context, vcs_utils, jsms, summary):
+ esms = []
+ for jsm in jsms:
+ esm = rename_single_file(command_context, vcs_utils, jsm, summary)
+ esms.append(esm)
+
+ return esms
+
+
+npm_prefix = pathlib.Path("tools") / "esmify"
+path_from_npm_prefix = pathlib.Path("..") / ".."
+
+
+def setup_jscodeshift():
+ """Install jscodeshift."""
+ cmd = [
+ sys.executable,
+ "./mach",
+ "npm",
+ "install",
+ "jscodeshift",
+ "--save-dev",
+ "--prefix",
+ str(npm_prefix),
+ ]
+ subprocess.run(cmd, check=True)
+
+
+def run_npm_command(args, env, stdin):
+ cmd = [
+ sys.executable,
+ "./mach",
+ "npm",
+ "run",
+ ] + args
+ p = subprocess.Popen(cmd, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+ p.stdin.write(stdin)
+ p.stdin.close()
+
+ ok_files = []
+ errors = []
+ while True:
+ line = p.stdout.readline()
+ if not line:
+ break
+ line = line.rstrip().decode()
+
+ if line.startswith(" NOC "):
+ continue
+
+ print(line)
+
+ m = re.search(r"^ (OKK|ERR) ([^ ]+)(?: (.+))?", line)
+ if not m:
+ continue
+
+ result = m.group(1)
+ # NOTE: path is written from `tools/esmify`.
+ path = pathlib.Path(m.group(2)).relative_to(path_from_npm_prefix)
+ error = m.group(3)
+
+ if result == "OKK":
+ ok_files.append(path)
+
+ if result == "ERR":
+ errors.append([path, error])
+
+ if p.wait() != 0:
+ return [None, None]
+
+ return ok_files, errors
+
+
+def convert_module(jsms, summary):
+ """Replace EXPORTED_SYMBOLS with export declarations, and replace
+ ChromeUtils.importESModule with static import as much as possible,
+ and return the list of successfully rewritten files."""
+
+ if len(jsms) == 0:
+ return []
+
+ env = os.environ.copy()
+
+ stdin = "\n".join(map(str, paths_from_npm_prefix(jsms))).encode()
+
+ ok_files, errors = run_npm_command(
+ [
+ "convert_module",
+ "--prefix",
+ str(npm_prefix),
+ ],
+ env=env,
+ stdin=stdin,
+ )
+
+ if ok_files is None and errors is None:
+ return None
+
+ summary.convert_errors.extend(errors)
+
+ return ok_files
+
+
+def rewrite_imports(jss, prefix, summary):
+ """Replace import calls for JSM with import calls for ESM or static import
+ for ESM."""
+
+ if len(jss) == 0:
+ return []
+
+ env = os.environ.copy()
+ env["ESMIFY_TARGET_PREFIX"] = prefix
+
+ stdin = "\n".join(map(str, paths_from_npm_prefix(jss))).encode()
+
+ ok_files, errors = run_npm_command(
+ [
+ "rewrite_imports",
+ "--prefix",
+ str(npm_prefix),
+ ],
+ env=env,
+ stdin=stdin,
+ )
+
+ if ok_files is None and errors is None:
+ return None
+
+ summary.import_errors.extend(errors)
+
+ return ok_files
+
+
+def paths_from_npm_prefix(paths):
+ """Convert relative path from mozilla-central to relative path from
+ tools/esmify."""
+ return list(map(lambda path: path_from_npm_prefix / path, paths))
+
+
+def eslint_fix(command_context, files):
+ """Auto format files."""
+
+ def info(text):
+ command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}")
+
+ if len(files) == 0:
+ return
+
+ remaining = files[0:]
+
+ # There can be too many files for single command line, perform by chunk.
+ max_files = 16
+ while len(remaining) > max_files:
+ info(f"{len(remaining)} files remaining")
+
+ chunk = remaining[0:max_files]
+ remaining = remaining[max_files:]
+
+ cmd = [sys.executable, "./mach", "eslint", "--fix"] + chunk
+ subprocess.run(cmd, check=True)
+
+ info(f"{len(remaining)} files remaining")
+ chunk = remaining
+ cmd = [sys.executable, "./mach", "eslint", "--fix"] + chunk
+ subprocess.run(cmd, check=True)