From d8bbc7858622b6d9c278469aab701ca0b609cddf Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 15 May 2024 05:35:49 +0200 Subject: Merging upstream version 126.0. Signed-off-by: Daniel Baumann --- python/mozbuild/mozbuild/action/webidl.py | 6 ++ python/mozbuild/mozbuild/artifacts.py | 68 +++++++++++++--------- .../mozbuild/mozbuild/backend/cargo_build_defs.py | 48 --------------- python/mozbuild/mozbuild/backend/recursivemake.py | 13 +++++ python/mozbuild/mozbuild/build_commands.py | 6 +- python/mozbuild/mozbuild/compilation/database.py | 2 +- python/mozbuild/mozbuild/frontend/context.py | 8 +-- python/mozbuild/mozbuild/generated_sources.py | 12 ++++ python/mozbuild/mozbuild/mach_commands.py | 7 ++- python/mozbuild/mozbuild/schedules.py | 4 ++ python/mozbuild/mozbuild/shellutil.py | 4 +- .../mozbuild/test/backend/test_recursivemake.py | 28 ++++++++- .../configure/macos_fake_sdk/SDKSettings.plist | 2 +- .../mozbuild/test/configure/test_moz_configure.py | 14 ++++- python/mozbuild/mozbuild/vendor/moz_yaml.py | 11 +++- python/mozbuild/mozbuild/vendor/vendor_manifest.py | 17 ++++++ python/mozbuild/mozbuild/vendor/vendor_rust.py | 9 +-- python/mozbuild/mozpack/files.py | 2 +- 18 files changed, 162 insertions(+), 99 deletions(-) (limited to 'python/mozbuild') diff --git a/python/mozbuild/mozbuild/action/webidl.py b/python/mozbuild/mozbuild/action/webidl.py index ade25b4c0c..8a9bc5cb2b 100644 --- a/python/mozbuild/mozbuild/action/webidl.py +++ b/python/mozbuild/mozbuild/action/webidl.py @@ -14,4 +14,10 @@ def main(argv): if __name__ == "__main__": + # When run from the CLI, we know we have a rather short duration so it's + # fine to disable gc in order to shave a few milliseconds. + import gc + + gc.disable() + sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/artifacts.py b/python/mozbuild/mozbuild/artifacts.py index a4d186816a..eff16f6c9b 100644 --- a/python/mozbuild/mozbuild/artifacts.py +++ b/python/mozbuild/mozbuild/artifacts.py @@ -646,26 +646,38 @@ class MacArtifactJob(ArtifactJob): # These get copied into dist/bin without the path, so "root/a/b/c" -> "dist/bin/c". _paths_no_keep_path = ( - "Contents/MacOS", - [ - "crashreporter.app/Contents/MacOS/crashreporter", - "{product}", - "{product}-bin", - "*.dylib", - "minidump-analyzer", - "nmhproxy", - "pingsender", - "plugin-container.app/Contents/MacOS/plugin-container", - "updater.app/Contents/MacOS/org.mozilla.updater", - # 'xpcshell', - "XUL", - ], + ( + "Contents/MacOS", + [ + "crashreporter.app/Contents/MacOS/crashreporter", + "{product}", + "{product}-bin", + "*.dylib", + "minidump-analyzer", + "nmhproxy", + "pingsender", + "plugin-container.app/Contents/MacOS/plugin-container", + "updater.app/Contents/Frameworks/UpdateSettings.framework/UpdateSettings", + "updater.app/Contents/MacOS/org.mozilla.updater", + # 'xpcshell', + "XUL", + ], + ), + ( + "Contents/Frameworks", + [ + "ChannelPrefs.framework/ChannelPrefs", + ], + ), ) @property def paths_no_keep_path(self): - root, paths = self._paths_no_keep_path - return (root, [p.format(product=self.product) for p in paths]) + formatted = [] + for root, paths in self._paths_no_keep_path: + formatted.append((root, [p.format(product=self.product) for p in paths])) + + return tuple(formatted) @contextmanager def get_writer(self, **kwargs): @@ -726,18 +738,18 @@ class MacArtifactJob(ArtifactJob): ] with self.get_writer(file=processed_filename, compress_level=5) as writer: - root, paths = self.paths_no_keep_path - finder = UnpackFinder(mozpath.join(source, root)) - for path in paths: - for p, f in finder.find(path): - self.log( - logging.DEBUG, - "artifact", - {"path": p}, - "Adding {path} to processed archive", - ) - destpath = mozpath.join("bin", os.path.basename(p)) - writer.add(destpath.encode("utf-8"), f.open(), mode=f.mode) + for root, paths in self.paths_no_keep_path: + finder = UnpackFinder(mozpath.join(source, root)) + for path in paths: + for p, f in finder.find(path): + self.log( + logging.DEBUG, + "artifact", + {"path": p}, + "Adding {path} to processed archive", + ) + destpath = mozpath.join("bin", os.path.basename(p)) + writer.add(destpath.encode("utf-8"), f.open(), mode=f.mode) for root, paths in paths_keep_path: finder = UnpackFinder(mozpath.join(source, root)) diff --git a/python/mozbuild/mozbuild/backend/cargo_build_defs.py b/python/mozbuild/mozbuild/backend/cargo_build_defs.py index c60fd2abf6..1662a5f97f 100644 --- a/python/mozbuild/mozbuild/backend/cargo_build_defs.py +++ b/python/mozbuild/mozbuild/backend/cargo_build_defs.py @@ -14,57 +14,9 @@ cargo_extra_outputs = { "selectors": ["ascii_case_insensitive_html_attributes.rs"], "style": [ "gecko/atom_macro.rs", - "gecko/bindings.rs", "gecko/pseudo_element_definition.rs", "gecko/structs.rs", - "gecko_properties.rs", - "longhands/background.rs", - "longhands/border.rs", - "longhands/box.rs", - "longhands/color.rs", - "longhands/column.rs", - "longhands/counters.rs", - "longhands/effects.rs", - "longhands/font.rs", - "longhands/inherited_box.rs", - "longhands/inherited_svg.rs", - "longhands/inherited_table.rs", - "longhands/inherited_text.rs", - "longhands/inherited_ui.rs", - "longhands/list.rs", - "longhands/margin.rs", - "longhands/outline.rs", - "longhands/padding.rs", - "longhands/position.rs", - "longhands/svg.rs", - "longhands/table.rs", - "longhands/text.rs", - "longhands/ui.rs", - "longhands/xul.rs", "properties.rs", - "shorthands/background.rs", - "shorthands/border.rs", - "shorthands/box.rs", - "shorthands/color.rs", - "shorthands/column.rs", - "shorthands/counters.rs", - "shorthands/effects.rs", - "shorthands/font.rs", - "shorthands/inherited_box.rs", - "shorthands/inherited_svg.rs", - "shorthands/inherited_table.rs", - "shorthands/inherited_text.rs", - "shorthands/inherited_ui.rs", - "shorthands/list.rs", - "shorthands/margin.rs", - "shorthands/outline.rs", - "shorthands/padding.rs", - "shorthands/position.rs", - "shorthands/svg.rs", - "shorthands/table.rs", - "shorthands/text.rs", - "shorthands/ui.rs", - "shorthands/xul.rs", ], "webrender": ["shaders.rs"], "geckodriver": ["build-info.rs"], diff --git a/python/mozbuild/mozbuild/backend/recursivemake.py b/python/mozbuild/mozbuild/backend/recursivemake.py index 1180d9976e..86cb6ccc10 100644 --- a/python/mozbuild/mozbuild/backend/recursivemake.py +++ b/python/mozbuild/mozbuild/backend/recursivemake.py @@ -1840,6 +1840,9 @@ class RecursiveMakeBackend(MakeBackend): ) ) + ipdl_srcs_path = mozpath.join(ipdl_dir, "ipdlsrcs.txt") + mk.add_statement("ALL_IPDLSRCS_FILE := {}".format(ipdl_srcs_path)) + # Preprocessed ipdl files are generated in ipdl_dir. mk.add_statement( "IPDLDIRS := %s %s" @@ -1851,6 +1854,16 @@ class RecursiveMakeBackend(MakeBackend): ) ) + # Bug 1885948: Passing all of these filenames directly to ipdl.py as + # command-line arguments can go over the maximum argument length on + # Windows (32768 bytes) if the checkout path is sufficiently long. + with self._write_file(ipdl_srcs_path) as srcs: + for filename in sorted_nonstatic_ipdl_basenames: + srcs.write("{}\n".format(filename)) + + for filename in sorted_static_ipdl_sources: + srcs.write("{}\n".format(filename)) + with self._write_file(mozpath.join(ipdl_dir, "ipdlsrcs.mk")) as ipdls: mk.dump(ipdls, removal_guard=False) diff --git a/python/mozbuild/mozbuild/build_commands.py b/python/mozbuild/mozbuild/build_commands.py index 3299ca712e..563c726100 100644 --- a/python/mozbuild/mozbuild/build_commands.py +++ b/python/mozbuild/mozbuild/build_commands.py @@ -113,10 +113,10 @@ def _set_priority(priority, verbose): ) @CommandArgument( "--priority", - default="less", + default="idle", metavar="priority", type=str, - help="idle/less/normal/more/high. (Default less)", + help="idle/less/normal/more/high. (Default idle)", ) def build( command_context, @@ -126,7 +126,7 @@ def build( directory=None, verbose=False, keep_going=False, - priority="less", + priority="idle", ): """Build the source tree. diff --git a/python/mozbuild/mozbuild/compilation/database.py b/python/mozbuild/mozbuild/compilation/database.py index e741c88a81..51e2faa99f 100644 --- a/python/mozbuild/mozbuild/compilation/database.py +++ b/python/mozbuild/mozbuild/compilation/database.py @@ -132,7 +132,7 @@ class CompileDBBackend(CommonBackend): db.append( { "directory": directory, - "command": " ".join(shell_quote(a) for a in c), + "command": shell_quote(*c), "file": mozpath.join(directory, filename), } ) diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozbuild/frontend/context.py index 551ea00feb..a3fe62710b 100644 --- a/python/mozbuild/mozbuild/frontend/context.py +++ b/python/mozbuild/mozbuild/frontend/context.py @@ -1223,8 +1223,8 @@ class Files(SubContext): ``foo.html`` Will match only the ``foo.html`` file in the current directory. - ``*.jsm`` - Will match all ``.jsm`` files in the current directory. + ``*.mjs`` + Will match all ``.mjs`` files in the current directory. ``**/*.cpp`` Will match all ``.cpp`` files in this and all child directories. ``foo/*.css`` @@ -2943,13 +2943,13 @@ SPECIAL_VARIABLES = { list, """JavaScript modules to install in the test-only destination. - Some JavaScript modules (JSMs) are test-only and not distributed + Some JavaScript modules are test-only and not distributed with Firefox. This variable defines them. To install modules in a subdirectory, use properties of this variable to control the final destination. e.g. - ``TESTING_JS_MODULES.foo += ['module.jsm']``. + ``TESTING_JS_MODULES.foo += ['module.sys.mjs']``. """, ), "TEST_DIRS": ( diff --git a/python/mozbuild/mozbuild/generated_sources.py b/python/mozbuild/mozbuild/generated_sources.py index e22e71e5f6..81b6f4d6d8 100644 --- a/python/mozbuild/mozbuild/generated_sources.py +++ b/python/mozbuild/mozbuild/generated_sources.py @@ -58,6 +58,18 @@ def get_generated_sources(): for p, f in finder: if p.endswith(GENERATED_SOURCE_EXTS): yield mozpath.join(base, p), f + # Next, return source files that were generated by AIDL. + if buildconfig.substs.get("MOZ_WIDGET_TOOLKIT") == "android": + base = "gradle/build/mobile/android/geckoview/generated/aidl_source_output_dir" + finder = FileFinder(mozpath.join(buildconfig.topobjdir, base)) + found = False + for p, f in finder.find("**/*.java"): + found = True + yield mozpath.join(base, p), f + if not found: + raise Exception( + "No AIDL-generated files are found. Maybe the output directory has changed?" + ) def get_s3_region_and_bucket(): diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py index 2398f8de03..20b451dc16 100644 --- a/python/mozbuild/mozbuild/mach_commands.py +++ b/python/mozbuild/mozbuild/mach_commands.py @@ -390,11 +390,12 @@ def cargo_vet(command_context, arguments, stdout=None, env=os.environ): locked = "--locked" in arguments if locked: - # The use of --locked requires .cargo/config to exist, but other things, + # The use of --locked requires .cargo/config.toml to exist, but other things, # like cargo update, don't want it there, so remove it once we're done. topsrcdir = Path(command_context.topsrcdir) shutil.copyfile( - topsrcdir / ".cargo" / "config.in", topsrcdir / ".cargo" / "config" + topsrcdir / ".cargo" / "config.toml.in", + topsrcdir / ".cargo" / "config.toml", ) try: @@ -406,7 +407,7 @@ def cargo_vet(command_context, arguments, stdout=None, env=os.environ): ) finally: if locked: - (topsrcdir / ".cargo" / "config").unlink() + (topsrcdir / ".cargo" / "config.toml").unlink() # When the function is invoked without stdout set (the default when running # as a mach subcommand), exit with the returncode from cargo vet. diff --git a/python/mozbuild/mozbuild/schedules.py b/python/mozbuild/mozbuild/schedules.py index 0b7d9b1154..5cc2d0abdc 100644 --- a/python/mozbuild/mozbuild/schedules.py +++ b/python/mozbuild/mozbuild/schedules.py @@ -43,6 +43,10 @@ EXCLUSIVE_COMPONENTS = [ "macosx", "windows", "ios", + # application + "firefox", + "fenix", + "focus-android", # broad test harness categories "awsy", "condprofile", diff --git a/python/mozbuild/mozbuild/shellutil.py b/python/mozbuild/mozbuild/shellutil.py index fab3ae6086..65af11814a 100644 --- a/python/mozbuild/mozbuild/shellutil.py +++ b/python/mozbuild/mozbuild/shellutil.py @@ -182,10 +182,10 @@ def _quote(s): not enclosed in quotes. """ if type(s) == int: - return "%d" % s + return f"{s}" # Empty strings need to be quoted to have any significance - if s and not SHELL_QUOTE_RE.search(s) and not s.startswith("~"): + if s and not SHELL_QUOTE_RE.search(s) and s[0] != "~": return s # Single quoted strings can contain any characters unescaped except the diff --git a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py index 2d39308eb3..0706a302bf 100644 --- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py +++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py @@ -804,16 +804,40 @@ class TestRecursiveMakeBackend(BackendTester): # Handle Windows paths correctly topsrcdir = mozpath.normsep(env.topsrcdir) + ipdlsrcs_file_path = mozpath.join(ipdl_root, "ipdlsrcs.txt") + + ipdlsrcs = ["bar1.ipdl", "foo1.ipdl"] + ipdlsrcs.extend( + "%s/%s" % (topsrcdir, path) + for path in ( + "bar/bar.ipdl", + "bar/bar2.ipdlh", + "foo/foo.ipdl", + "foo/foo2.ipdlh", + ) + ) + self.maxDiff = None expected = [ - "ALL_IPDLSRCS := bar1.ipdl foo1.ipdl %s/bar/bar.ipdl %s/bar/bar2.ipdlh %s/foo/foo.ipdl %s/foo/foo2.ipdlh" # noqa - % tuple([topsrcdir] * 4), + "ALL_IPDLSRCS := %s" % (" ".join(ipdlsrcs)), + "ALL_IPDLSRCS_FILE := %s" % ipdlsrcs_file_path, "IPDLDIRS := %s %s/bar %s/foo" % (ipdl_root, topsrcdir, topsrcdir), ] found = [str for str in lines if str.startswith(("ALL_IPDLSRCS", "IPDLDIRS"))] self.assertEqual(found, expected) + # Check the ipdlsrcs.txt file was written correctly. + self.assertTrue(ipdlsrcs_file_path, "ipdlsrcs.txt was written") + with open(ipdlsrcs_file_path) as f: + ipdlsrcs_file_contents = f.read().splitlines() + + self.assertEqual( + ipdlsrcs_file_contents, + ipdlsrcs, + "ipdlsrcs.txt contains all IPDL sources", + ) + # Check that each directory declares the generated relevant .cpp files # to be built in CPPSRCS. # ENABLE_UNIFIED_BUILD defaults to False without mozilla-central's diff --git a/python/mozbuild/mozbuild/test/configure/macos_fake_sdk/SDKSettings.plist b/python/mozbuild/mozbuild/test/configure/macos_fake_sdk/SDKSettings.plist index 786746f103..ef035ec3f0 100644 --- a/python/mozbuild/mozbuild/test/configure/macos_fake_sdk/SDKSettings.plist +++ b/python/mozbuild/mozbuild/test/configure/macos_fake_sdk/SDKSettings.plist @@ -3,6 +3,6 @@ Version - 14.2 + 14.4 diff --git a/python/mozbuild/mozbuild/test/configure/test_moz_configure.py b/python/mozbuild/mozbuild/test/configure/test_moz_configure.py index 7bb1b927e0..c10adb3b47 100644 --- a/python/mozbuild/mozbuild/test/configure/test_moz_configure.py +++ b/python/mozbuild/mozbuild/test/configure/test_moz_configure.py @@ -28,6 +28,8 @@ class TargetTest(BaseConfigureTest): platform = "win32" elif "openbsd6" in self.HOST: platform = "openbsd6" + elif "apple-darwin" in self.HOST: + platform = "darwin" else: raise Exception("Missing platform for HOST {}".format(self.HOST)) sandbox = self.get_sandbox({}, {}, args, env, cls=sandbox_class(platform)) @@ -111,7 +113,7 @@ class TestTargetAndroid(TargetTest): def test_target(self): self.assertEqual( self.get_target(["--enable-project=mobile/android"]), - "arm-unknown-linux-androideabi", + "x86_64-unknown-linux-android", ) self.assertEqual( self.get_target(["--enable-project=mobile/android", "--target=i686"]), @@ -131,6 +133,16 @@ class TestTargetAndroid(TargetTest): ) +class TestTargetAndroidAppleSiliconHost(TargetTest): + HOST = "aarch64-apple-darwin" + + def test_target(self): + self.assertEqual( + self.get_target(["--enable-project=mobile/android"]), + "aarch64-unknown-linux-android", + ) + + class TestTargetOpenBSD(TargetTest): # config.guess returns amd64 on OpenBSD, which we need to pass through to # config.sub so that it canonicalizes to x86_64. diff --git a/python/mozbuild/mozbuild/vendor/moz_yaml.py b/python/mozbuild/mozbuild/vendor/moz_yaml.py index c094d22a2b..1e329d237c 100644 --- a/python/mozbuild/mozbuild/vendor/moz_yaml.py +++ b/python/mozbuild/mozbuild/vendor/moz_yaml.py @@ -57,7 +57,15 @@ VALID_LICENSES = [ "Unicode", # http://www.unicode.org/copyright.html ] -VALID_SOURCE_HOSTS = ["gitlab", "googlesource", "github", "angle", "codeberg", "git"] +VALID_SOURCE_HOSTS = [ + "gitlab", + "googlesource", + "github", + "angle", + "codeberg", + "git", + "yaml-dir", +] """ --- @@ -443,6 +451,7 @@ def _schema_1(): Length(min=1), In(VALID_SOURCE_HOSTS, msg="Unsupported Source Hosting"), ), + "source-host-path": str, "tracking": Match(r"^(commit|tag)$"), "release-artifact": All(str, Length(min=1)), "flavor": Match(r"^(regular|rust|individual-files)$"), diff --git a/python/mozbuild/mozbuild/vendor/vendor_manifest.py b/python/mozbuild/mozbuild/vendor/vendor_manifest.py index ad9564405e..7becdd4c2a 100644 --- a/python/mozbuild/mozbuild/vendor/vendor_manifest.py +++ b/python/mozbuild/mozbuild/vendor/vendor_manifest.py @@ -348,6 +348,23 @@ class VendorManifest(MozbuildObject): from mozbuild.vendor.host_codeberg import CodebergHost return CodebergHost(self.manifest) + elif self.manifest["vendoring"]["source-hosting"] == "yaml-dir": + import importlib.util + + modulename, classname = self.manifest["vendoring"][ + "source-host-path" + ].rsplit(".", 1) + spec = importlib.util.spec_from_file_location( + modulename, + os.path.join( + os.path.dirname(self.yaml_file), + modulename.replace(".", os.sep) + ".py", + ), + ) + module = importlib.util.module_from_spec(spec) + sys.modules[modulename] = module + spec.loader.exec_module(module) + return getattr(module, classname)(self.manifest) else: raise Exception( "Unknown source host: " + self.manifest["vendoring"]["source-hosting"] diff --git a/python/mozbuild/mozbuild/vendor/vendor_rust.py b/python/mozbuild/mozbuild/vendor/vendor_rust.py index 15b644cf99..d81f344f32 100644 --- a/python/mozbuild/mozbuild/vendor/vendor_rust.py +++ b/python/mozbuild/mozbuild/vendor/vendor_rust.py @@ -47,9 +47,9 @@ CARGO_CONFIG_TEMPLATE = """\ # Take advantage of the fact that cargo will treat lines starting with # # as comments to add preprocessing directives. This file can thus by copied -# as-is to $topsrcdir/.cargo/config with no preprocessing to be used there +# as-is to $topsrcdir/.cargo/config.toml with no preprocessing to be used there # (for e.g. independent tasks building rust code), or be preprocessed by -# the build system to produce a .cargo/config with the right content. +# the build system to produce a .cargo/config.toml with the right content. #define REPLACE_NAME {replace_name} #define VENDORED_DIRECTORY {directory} # We explicitly exclude the following section when preprocessing because @@ -275,6 +275,7 @@ Please commit or stash these changes before vendoring, or re-run with `--ignore- "ISC", "MIT", "MPL-2.0", + "Unicode-3.0", "Unicode-DFS-2016", "Unlicense", "Zlib", @@ -846,7 +847,7 @@ license file's hash. output = res.stdout.decode("UTF-8") # Get the snippet of configuration that cargo vendor outputs, and - # update .cargo/config with it. + # update .cargo/config.toml with it. # XXX(bug 1576765): Hopefully do something better after # https://github.com/rust-lang/cargo/issues/7280 is addressed. config = "\n".join( @@ -880,7 +881,7 @@ license file's hash. mozpath.normsep(os.path.normcase(self.topsrcdir)), ) - cargo_config = os.path.join(self.topsrcdir, ".cargo", "config.in") + cargo_config = os.path.join(self.topsrcdir, ".cargo", "config.toml.in") with open(cargo_config, "w", encoding="utf-8", newline="\n") as fh: fh.write( CARGO_CONFIG_TEMPLATE.format( diff --git a/python/mozbuild/mozpack/files.py b/python/mozbuild/mozpack/files.py index a320e1f4b4..9ae16d5c1c 100644 --- a/python/mozbuild/mozpack/files.py +++ b/python/mozbuild/mozpack/files.py @@ -901,7 +901,7 @@ class BaseFinder(object): if path.endswith((".ftl", ".properties")): return MinifiedCommentStripped(file) - if self._minify_js and path.endswith((".js", ".jsm")): + if self._minify_js and path.endswith((".js", ".jsm", ".mjs")): return MinifiedJavaScript(file, self._minify_js_verify_command) return file -- cgit v1.2.3