diff options
Diffstat (limited to 'python')
29 files changed, 484 insertions, 131 deletions
diff --git a/python/l10n/fluent_migrations/bug_1759175_crashreporter.py b/python/l10n/fluent_migrations/bug_1759175_crashreporter.py new file mode 100644 index 0000000000..245799b426 --- /dev/null +++ b/python/l10n/fluent_migrations/bug_1759175_crashreporter.py @@ -0,0 +1,127 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +import fluent.syntax.ast as FTL +from fluent.migrate.helpers import transforms_from, VARIABLE_REFERENCE, TERM_REFERENCE +from fluent.migrate.transforms import ( + CONCAT, + REPLACE, + REPLACE_IN_TEXT, + LegacySource, + Transform, +) + + +class SPLIT_AND_REPLACE(LegacySource): + """Split sentence on '\n\n', return a specific segment, and perform replacements if needed""" + + def __init__(self, path, key, index, replacements=None, **kwargs): + super(SPLIT_AND_REPLACE, self).__init__(path, key, **kwargs) + self.index = index + self.replacements = replacements + + def __call__(self, ctx): + element = super(SPLIT_AND_REPLACE, self).__call__(ctx) + segments = element.value.split(r"\n\n") + element.value = segments[self.index] + + if self.replacements is None: + return Transform.pattern_of(element) + else: + return REPLACE_IN_TEXT(element, self.replacements)(ctx) + + +def migrate(ctx): + """Bug 1759175 - Migrate Crash Reporter to Fluent, part {index}.""" + + target_file = "toolkit/crashreporter/crashreporter.ftl" + source_file = "toolkit/crashreporter/crashreporter.ini" + + ctx.add_transforms( + target_file, + target_file, + transforms_from( + """ +crashreporter-title = { COPY(from_path, "CrashReporterTitle") } +crashreporter-no-run-message = { COPY(from_path, "CrashReporterDefault") } +crashreporter-button-details = { COPY(from_path, "Details") } +crashreporter-view-report-title = { COPY(from_path, "ViewReportTitle") } +crashreporter-comment-prompt = { COPY(from_path, "CommentGrayText") } +crashreporter-report-info = { COPY(from_path, "ExtraReportInfo") } +crashreporter-submit-status = { COPY(from_path, "ReportPreSubmit2") } +crashreporter-submit-in-progress = { COPY(from_path, "ReportDuringSubmit2") } +crashreporter-submit-success = { COPY(from_path, "ReportSubmitSuccess") } +crashreporter-submit-failure = { COPY(from_path, "ReportSubmitFailed") } +crashreporter-resubmit-status = { COPY(from_path, "ReportResubmit") } +crashreporter-button-ok = { COPY(from_path, "Ok") } +crashreporter-button-close = { COPY(from_path, "Close") } + """, + from_path=source_file, + ), + ) + ctx.add_transforms( + target_file, + target_file, + [ + FTL.Message( + id=FTL.Identifier("crashreporter-crash-message"), + value=SPLIT_AND_REPLACE( + source_file, + "CrashReporterDescriptionText2", + index=0, + replacements={ + "%s": TERM_REFERENCE("brand-short-name"), + }, + ), + ), + FTL.Message( + id=FTL.Identifier("crashreporter-plea"), + value=SPLIT_AND_REPLACE( + source_file, + "CrashReporterDescriptionText2", + index=1, + ), + ), + FTL.Message( + id=FTL.Identifier("crashreporter-error-details"), + value=SPLIT_AND_REPLACE( + source_file, + "CrashReporterProductErrorText2", + index=2, + replacements={ + "%s": VARIABLE_REFERENCE("details"), + }, + ), + ), + FTL.Message( + id=FTL.Identifier("crashreporter-button-quit"), + value=REPLACE( + source_file, + "Quit2", + { + "%s": TERM_REFERENCE("brand-short-name"), + }, + ), + ), + FTL.Message( + id=FTL.Identifier("crashreporter-button-restart"), + value=REPLACE( + source_file, + "Restart", + { + "%s": TERM_REFERENCE("brand-short-name"), + }, + ), + ), + FTL.Message( + id=FTL.Identifier("crashreporter-crash-identifier"), + value=REPLACE( + source_file, + "CrashID", + { + "%s": VARIABLE_REFERENCE("id"), + }, + ), + ), + ], + ) diff --git a/python/l10n/fluent_migrations/bug_1864038_screenshots_migration.py b/python/l10n/fluent_migrations/bug_1864038_screenshots_migration.py new file mode 100644 index 0000000000..5427a9a848 --- /dev/null +++ b/python/l10n/fluent_migrations/bug_1864038_screenshots_migration.py @@ -0,0 +1,39 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +from fluent.migrate.helpers import transforms_from + + +def migrate(ctx): + """Bug 1864038 - Consolidate screenshots fluent files, part {index}.""" + + source = "browser/browser/screenshotsOverlay.ftl" + target = "browser/browser/screenshots.ftl" + + ctx.add_transforms( + target, + target, + transforms_from( + """ +screenshots-component-copy-button-label = {COPY_PATTERN(from_path, "screenshots-overlay-copy-button")} + +screenshots-component-download-button-label = {COPY_PATTERN(from_path, "screenshots-overlay-download-button")} + +screenshots-overlay-selection-region-size-2 = {COPY_PATTERN(from_path, "screenshots-overlay-selection-region-size")} +""", + from_path=source, + ), + ) + + ctx.add_transforms( + target, + target, + transforms_from( + """ +screenshots-component-retry-button = + .title = {COPY_PATTERN(from_path, "screenshots-retry-button-title.title")} + .aria-label = {COPY_PATTERN(from_path, "screenshots-retry-button-title.title")} +""", + from_path=target, + ), + ) diff --git a/python/l10n/fluent_migrations/bug_1881582_autocomplete_merge.py b/python/l10n/fluent_migrations/bug_1881582_autocomplete_merge.py new file mode 100644 index 0000000000..3a013cf82b --- /dev/null +++ b/python/l10n/fluent_migrations/bug_1881582_autocomplete_merge.py @@ -0,0 +1,63 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +import fluent.syntax.ast as FTL +from fluent.migrate.helpers import VARIABLE_REFERENCE +from fluent.migrate.transforms import COPY, REPLACE + + +def migrate(ctx): + """Bug 1881582 - Merge form autofill autocomplete items into normal autocomplete UI part {index}.""" + + propertiesSource = "browser/extensions/formautofill/formautofill.properties" + target = "toolkit/toolkit/formautofill/formAutofill.ftl" + ctx.add_transforms( + target, + target, + [ + FTL.Message( + id=FTL.Identifier("autofill-clear-form-label"), + value=COPY(propertiesSource, "clearFormBtnLabel2"), + ), + FTL.Message( + id=FTL.Identifier("autofill-category-address"), + value=COPY(propertiesSource, "category.address"), + ), + FTL.Message( + id=FTL.Identifier("autofill-category-name"), + value=COPY(propertiesSource, "category.name"), + ), + FTL.Message( + id=FTL.Identifier("autofill-category-organization"), + value=COPY(propertiesSource, "category.organization2"), + ), + FTL.Message( + id=FTL.Identifier("autofill-category-tel"), + value=COPY(propertiesSource, "category.tel"), + ), + FTL.Message( + id=FTL.Identifier("autofill-category-email"), + value=COPY(propertiesSource, "category.email"), + ), + FTL.Message( + id=FTL.Identifier("autofill-phishing-warningmessage-extracategory"), + value=REPLACE( + propertiesSource, + "phishingWarningMessage", + { + "%1$S": VARIABLE_REFERENCE("categories"), + }, + ), + ), + FTL.Message( + id=FTL.Identifier("autofill-phishing-warningmessage"), + value=REPLACE( + propertiesSource, + "phishingWarningMessage2", + { + "%1$S": VARIABLE_REFERENCE("categories"), + }, + ), + ), + ], + ) diff --git a/python/l10n/fluent_migrations/bug_1884748_alt_text_images_sync.py b/python/l10n/fluent_migrations/bug_1884748_alt_text_images_sync.py new file mode 100644 index 0000000000..924b31c6c1 --- /dev/null +++ b/python/l10n/fluent_migrations/bug_1884748_alt_text_images_sync.py @@ -0,0 +1,24 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +from fluent.migrate.helpers import transforms_from + + +def migrate(ctx): + """Bug 1884748 - Add alt text to images in about:preferences#sync, part {index}.""" + + source = "browser/browser/preferences/preferences.ftl" + target = source + + ctx.add_transforms( + target, + target, + transforms_from( + """ +sync-profile-picture-with-alt = + .tooltiptext = {COPY_PATTERN(from_path, "sync-profile-picture.tooltiptext")} + .alt = {COPY_PATTERN(from_path, "sync-profile-picture.tooltiptext")} +""", + from_path=source, + ), + ) diff --git a/python/mach/mach/command_util.py b/python/mach/mach/command_util.py index e8238bd83e..2c35c8b01c 100644 --- a/python/mach/mach/command_util.py +++ b/python/mach/mach/command_util.py @@ -48,6 +48,7 @@ class MachCommandReference: MACH_COMMANDS = { + "addstory": MachCommandReference("toolkit/content/widgets/mach_commands.py"), "addtest": MachCommandReference("testing/mach_commands.py"), "addwidget": MachCommandReference("toolkit/content/widgets/mach_commands.py"), "android": MachCommandReference("mobile/android/mach_commands.py"), @@ -67,6 +68,7 @@ MACH_COMMANDS = { "python/mozbuild/mozbuild/build_commands.py", ), "buildsymbols": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), + "buildtokens": MachCommandReference("toolkit/content/widgets/mach_commands.py"), "busted": MachCommandReference("tools/mach_commands.py"), "cargo": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), "clang-format": MachCommandReference( @@ -92,6 +94,9 @@ MACH_COMMANDS = { "environment": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), "eslint": MachCommandReference("tools/lint/mach_commands.py"), "esmify": MachCommandReference("tools/esmify/mach_commands.py"), + "event-into-legacy": MachCommandReference( + "toolkit/components/glean/build_scripts/mach_commands.py" + ), "fetch-condprofile": MachCommandReference("testing/condprofile/mach_commands.py"), "file-info": MachCommandReference( "python/mozbuild/mozbuild/frontend/mach_commands.py" diff --git a/python/mozboot/mozboot/bootstrap.py b/python/mozboot/mozboot/bootstrap.py index 89ddcdf5a2..caf2821dac 100644 --- a/python/mozboot/mozboot/bootstrap.py +++ b/python/mozboot/mozboot/bootstrap.py @@ -633,7 +633,12 @@ def update_mercurial_repo(hg: Path, url, dest: Path, revision): print(f"Ensuring {url} is up to date at {dest}") env = os.environ.copy() - env.update({"HGPLAIN": "1"}) + env.update( + { + "HGPLAIN": "1", + "HGRCPATH": "!", + } + ) try: subprocess.check_call(pull_args, cwd=str(cwd), env=env) diff --git a/python/mozbuild/mozbuild/action/webidl.py b/python/mozbuild/mozbuild/action/webidl.py index ade25b4c0c..8a9bc5cb2b 100644 --- a/python/mozbuild/mozbuild/action/webidl.py +++ b/python/mozbuild/mozbuild/action/webidl.py @@ -14,4 +14,10 @@ def main(argv): if __name__ == "__main__": + # When run from the CLI, we know we have a rather short duration so it's + # fine to disable gc in order to shave a few milliseconds. + import gc + + gc.disable() + sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/artifacts.py b/python/mozbuild/mozbuild/artifacts.py index a4d186816a..eff16f6c9b 100644 --- a/python/mozbuild/mozbuild/artifacts.py +++ b/python/mozbuild/mozbuild/artifacts.py @@ -646,26 +646,38 @@ class MacArtifactJob(ArtifactJob): # These get copied into dist/bin without the path, so "root/a/b/c" -> "dist/bin/c". _paths_no_keep_path = ( - "Contents/MacOS", - [ - "crashreporter.app/Contents/MacOS/crashreporter", - "{product}", - "{product}-bin", - "*.dylib", - "minidump-analyzer", - "nmhproxy", - "pingsender", - "plugin-container.app/Contents/MacOS/plugin-container", - "updater.app/Contents/MacOS/org.mozilla.updater", - # 'xpcshell', - "XUL", - ], + ( + "Contents/MacOS", + [ + "crashreporter.app/Contents/MacOS/crashreporter", + "{product}", + "{product}-bin", + "*.dylib", + "minidump-analyzer", + "nmhproxy", + "pingsender", + "plugin-container.app/Contents/MacOS/plugin-container", + "updater.app/Contents/Frameworks/UpdateSettings.framework/UpdateSettings", + "updater.app/Contents/MacOS/org.mozilla.updater", + # 'xpcshell', + "XUL", + ], + ), + ( + "Contents/Frameworks", + [ + "ChannelPrefs.framework/ChannelPrefs", + ], + ), ) @property def paths_no_keep_path(self): - root, paths = self._paths_no_keep_path - return (root, [p.format(product=self.product) for p in paths]) + formatted = [] + for root, paths in self._paths_no_keep_path: + formatted.append((root, [p.format(product=self.product) for p in paths])) + + return tuple(formatted) @contextmanager def get_writer(self, **kwargs): @@ -726,18 +738,18 @@ class MacArtifactJob(ArtifactJob): ] with self.get_writer(file=processed_filename, compress_level=5) as writer: - root, paths = self.paths_no_keep_path - finder = UnpackFinder(mozpath.join(source, root)) - for path in paths: - for p, f in finder.find(path): - self.log( - logging.DEBUG, - "artifact", - {"path": p}, - "Adding {path} to processed archive", - ) - destpath = mozpath.join("bin", os.path.basename(p)) - writer.add(destpath.encode("utf-8"), f.open(), mode=f.mode) + for root, paths in self.paths_no_keep_path: + finder = UnpackFinder(mozpath.join(source, root)) + for path in paths: + for p, f in finder.find(path): + self.log( + logging.DEBUG, + "artifact", + {"path": p}, + "Adding {path} to processed archive", + ) + destpath = mozpath.join("bin", os.path.basename(p)) + writer.add(destpath.encode("utf-8"), f.open(), mode=f.mode) for root, paths in paths_keep_path: finder = UnpackFinder(mozpath.join(source, root)) diff --git a/python/mozbuild/mozbuild/backend/cargo_build_defs.py b/python/mozbuild/mozbuild/backend/cargo_build_defs.py index c60fd2abf6..1662a5f97f 100644 --- a/python/mozbuild/mozbuild/backend/cargo_build_defs.py +++ b/python/mozbuild/mozbuild/backend/cargo_build_defs.py @@ -14,57 +14,9 @@ cargo_extra_outputs = { "selectors": ["ascii_case_insensitive_html_attributes.rs"], "style": [ "gecko/atom_macro.rs", - "gecko/bindings.rs", "gecko/pseudo_element_definition.rs", "gecko/structs.rs", - "gecko_properties.rs", - "longhands/background.rs", - "longhands/border.rs", - "longhands/box.rs", - "longhands/color.rs", - "longhands/column.rs", - "longhands/counters.rs", - "longhands/effects.rs", - "longhands/font.rs", - "longhands/inherited_box.rs", - "longhands/inherited_svg.rs", - "longhands/inherited_table.rs", - "longhands/inherited_text.rs", - "longhands/inherited_ui.rs", - "longhands/list.rs", - "longhands/margin.rs", - "longhands/outline.rs", - "longhands/padding.rs", - "longhands/position.rs", - "longhands/svg.rs", - "longhands/table.rs", - "longhands/text.rs", - "longhands/ui.rs", - "longhands/xul.rs", "properties.rs", - "shorthands/background.rs", - "shorthands/border.rs", - "shorthands/box.rs", - "shorthands/color.rs", - "shorthands/column.rs", - "shorthands/counters.rs", - "shorthands/effects.rs", - "shorthands/font.rs", - "shorthands/inherited_box.rs", - "shorthands/inherited_svg.rs", - "shorthands/inherited_table.rs", - "shorthands/inherited_text.rs", - "shorthands/inherited_ui.rs", - "shorthands/list.rs", - "shorthands/margin.rs", - "shorthands/outline.rs", - "shorthands/padding.rs", - "shorthands/position.rs", - "shorthands/svg.rs", - "shorthands/table.rs", - "shorthands/text.rs", - "shorthands/ui.rs", - "shorthands/xul.rs", ], "webrender": ["shaders.rs"], "geckodriver": ["build-info.rs"], diff --git a/python/mozbuild/mozbuild/backend/recursivemake.py b/python/mozbuild/mozbuild/backend/recursivemake.py index 1180d9976e..86cb6ccc10 100644 --- a/python/mozbuild/mozbuild/backend/recursivemake.py +++ b/python/mozbuild/mozbuild/backend/recursivemake.py @@ -1840,6 +1840,9 @@ class RecursiveMakeBackend(MakeBackend): ) ) + ipdl_srcs_path = mozpath.join(ipdl_dir, "ipdlsrcs.txt") + mk.add_statement("ALL_IPDLSRCS_FILE := {}".format(ipdl_srcs_path)) + # Preprocessed ipdl files are generated in ipdl_dir. mk.add_statement( "IPDLDIRS := %s %s" @@ -1851,6 +1854,16 @@ class RecursiveMakeBackend(MakeBackend): ) ) + # Bug 1885948: Passing all of these filenames directly to ipdl.py as + # command-line arguments can go over the maximum argument length on + # Windows (32768 bytes) if the checkout path is sufficiently long. + with self._write_file(ipdl_srcs_path) as srcs: + for filename in sorted_nonstatic_ipdl_basenames: + srcs.write("{}\n".format(filename)) + + for filename in sorted_static_ipdl_sources: + srcs.write("{}\n".format(filename)) + with self._write_file(mozpath.join(ipdl_dir, "ipdlsrcs.mk")) as ipdls: mk.dump(ipdls, removal_guard=False) diff --git a/python/mozbuild/mozbuild/build_commands.py b/python/mozbuild/mozbuild/build_commands.py index 3299ca712e..563c726100 100644 --- a/python/mozbuild/mozbuild/build_commands.py +++ b/python/mozbuild/mozbuild/build_commands.py @@ -113,10 +113,10 @@ def _set_priority(priority, verbose): ) @CommandArgument( "--priority", - default="less", + default="idle", metavar="priority", type=str, - help="idle/less/normal/more/high. (Default less)", + help="idle/less/normal/more/high. (Default idle)", ) def build( command_context, @@ -126,7 +126,7 @@ def build( directory=None, verbose=False, keep_going=False, - priority="less", + priority="idle", ): """Build the source tree. diff --git a/python/mozbuild/mozbuild/compilation/database.py b/python/mozbuild/mozbuild/compilation/database.py index e741c88a81..51e2faa99f 100644 --- a/python/mozbuild/mozbuild/compilation/database.py +++ b/python/mozbuild/mozbuild/compilation/database.py @@ -132,7 +132,7 @@ class CompileDBBackend(CommonBackend): db.append( { "directory": directory, - "command": " ".join(shell_quote(a) for a in c), + "command": shell_quote(*c), "file": mozpath.join(directory, filename), } ) diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozbuild/frontend/context.py index 551ea00feb..a3fe62710b 100644 --- a/python/mozbuild/mozbuild/frontend/context.py +++ b/python/mozbuild/mozbuild/frontend/context.py @@ -1223,8 +1223,8 @@ class Files(SubContext): ``foo.html`` Will match only the ``foo.html`` file in the current directory. - ``*.jsm`` - Will match all ``.jsm`` files in the current directory. + ``*.mjs`` + Will match all ``.mjs`` files in the current directory. ``**/*.cpp`` Will match all ``.cpp`` files in this and all child directories. ``foo/*.css`` @@ -2943,13 +2943,13 @@ SPECIAL_VARIABLES = { list, """JavaScript modules to install in the test-only destination. - Some JavaScript modules (JSMs) are test-only and not distributed + Some JavaScript modules are test-only and not distributed with Firefox. This variable defines them. To install modules in a subdirectory, use properties of this variable to control the final destination. e.g. - ``TESTING_JS_MODULES.foo += ['module.jsm']``. + ``TESTING_JS_MODULES.foo += ['module.sys.mjs']``. """, ), "TEST_DIRS": ( diff --git a/python/mozbuild/mozbuild/generated_sources.py b/python/mozbuild/mozbuild/generated_sources.py index e22e71e5f6..81b6f4d6d8 100644 --- a/python/mozbuild/mozbuild/generated_sources.py +++ b/python/mozbuild/mozbuild/generated_sources.py @@ -58,6 +58,18 @@ def get_generated_sources(): for p, f in finder: if p.endswith(GENERATED_SOURCE_EXTS): yield mozpath.join(base, p), f + # Next, return source files that were generated by AIDL. + if buildconfig.substs.get("MOZ_WIDGET_TOOLKIT") == "android": + base = "gradle/build/mobile/android/geckoview/generated/aidl_source_output_dir" + finder = FileFinder(mozpath.join(buildconfig.topobjdir, base)) + found = False + for p, f in finder.find("**/*.java"): + found = True + yield mozpath.join(base, p), f + if not found: + raise Exception( + "No AIDL-generated files are found. Maybe the output directory has changed?" + ) def get_s3_region_and_bucket(): diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py index 2398f8de03..20b451dc16 100644 --- a/python/mozbuild/mozbuild/mach_commands.py +++ b/python/mozbuild/mozbuild/mach_commands.py @@ -390,11 +390,12 @@ def cargo_vet(command_context, arguments, stdout=None, env=os.environ): locked = "--locked" in arguments if locked: - # The use of --locked requires .cargo/config to exist, but other things, + # The use of --locked requires .cargo/config.toml to exist, but other things, # like cargo update, don't want it there, so remove it once we're done. topsrcdir = Path(command_context.topsrcdir) shutil.copyfile( - topsrcdir / ".cargo" / "config.in", topsrcdir / ".cargo" / "config" + topsrcdir / ".cargo" / "config.toml.in", + topsrcdir / ".cargo" / "config.toml", ) try: @@ -406,7 +407,7 @@ def cargo_vet(command_context, arguments, stdout=None, env=os.environ): ) finally: if locked: - (topsrcdir / ".cargo" / "config").unlink() + (topsrcdir / ".cargo" / "config.toml").unlink() # When the function is invoked without stdout set (the default when running # as a mach subcommand), exit with the returncode from cargo vet. diff --git a/python/mozbuild/mozbuild/schedules.py b/python/mozbuild/mozbuild/schedules.py index 0b7d9b1154..5cc2d0abdc 100644 --- a/python/mozbuild/mozbuild/schedules.py +++ b/python/mozbuild/mozbuild/schedules.py @@ -43,6 +43,10 @@ EXCLUSIVE_COMPONENTS = [ "macosx", "windows", "ios", + # application + "firefox", + "fenix", + "focus-android", # broad test harness categories "awsy", "condprofile", diff --git a/python/mozbuild/mozbuild/shellutil.py b/python/mozbuild/mozbuild/shellutil.py index fab3ae6086..65af11814a 100644 --- a/python/mozbuild/mozbuild/shellutil.py +++ b/python/mozbuild/mozbuild/shellutil.py @@ -182,10 +182,10 @@ def _quote(s): not enclosed in quotes. """ if type(s) == int: - return "%d" % s + return f"{s}" # Empty strings need to be quoted to have any significance - if s and not SHELL_QUOTE_RE.search(s) and not s.startswith("~"): + if s and not SHELL_QUOTE_RE.search(s) and s[0] != "~": return s # Single quoted strings can contain any characters unescaped except the diff --git a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py index 2d39308eb3..0706a302bf 100644 --- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py +++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py @@ -804,16 +804,40 @@ class TestRecursiveMakeBackend(BackendTester): # Handle Windows paths correctly topsrcdir = mozpath.normsep(env.topsrcdir) + ipdlsrcs_file_path = mozpath.join(ipdl_root, "ipdlsrcs.txt") + + ipdlsrcs = ["bar1.ipdl", "foo1.ipdl"] + ipdlsrcs.extend( + "%s/%s" % (topsrcdir, path) + for path in ( + "bar/bar.ipdl", + "bar/bar2.ipdlh", + "foo/foo.ipdl", + "foo/foo2.ipdlh", + ) + ) + self.maxDiff = None expected = [ - "ALL_IPDLSRCS := bar1.ipdl foo1.ipdl %s/bar/bar.ipdl %s/bar/bar2.ipdlh %s/foo/foo.ipdl %s/foo/foo2.ipdlh" # noqa - % tuple([topsrcdir] * 4), + "ALL_IPDLSRCS := %s" % (" ".join(ipdlsrcs)), + "ALL_IPDLSRCS_FILE := %s" % ipdlsrcs_file_path, "IPDLDIRS := %s %s/bar %s/foo" % (ipdl_root, topsrcdir, topsrcdir), ] found = [str for str in lines if str.startswith(("ALL_IPDLSRCS", "IPDLDIRS"))] self.assertEqual(found, expected) + # Check the ipdlsrcs.txt file was written correctly. + self.assertTrue(ipdlsrcs_file_path, "ipdlsrcs.txt was written") + with open(ipdlsrcs_file_path) as f: + ipdlsrcs_file_contents = f.read().splitlines() + + self.assertEqual( + ipdlsrcs_file_contents, + ipdlsrcs, + "ipdlsrcs.txt contains all IPDL sources", + ) + # Check that each directory declares the generated relevant .cpp files # to be built in CPPSRCS. # ENABLE_UNIFIED_BUILD defaults to False without mozilla-central's diff --git a/python/mozbuild/mozbuild/test/configure/macos_fake_sdk/SDKSettings.plist b/python/mozbuild/mozbuild/test/configure/macos_fake_sdk/SDKSettings.plist index 786746f103..ef035ec3f0 100644 --- a/python/mozbuild/mozbuild/test/configure/macos_fake_sdk/SDKSettings.plist +++ b/python/mozbuild/mozbuild/test/configure/macos_fake_sdk/SDKSettings.plist @@ -3,6 +3,6 @@ <plist version="1.0"> <dict> <key>Version</key> - <string>14.2</string> + <string>14.4</string> </dict> </plist> diff --git a/python/mozbuild/mozbuild/test/configure/test_moz_configure.py b/python/mozbuild/mozbuild/test/configure/test_moz_configure.py index 7bb1b927e0..c10adb3b47 100644 --- a/python/mozbuild/mozbuild/test/configure/test_moz_configure.py +++ b/python/mozbuild/mozbuild/test/configure/test_moz_configure.py @@ -28,6 +28,8 @@ class TargetTest(BaseConfigureTest): platform = "win32" elif "openbsd6" in self.HOST: platform = "openbsd6" + elif "apple-darwin" in self.HOST: + platform = "darwin" else: raise Exception("Missing platform for HOST {}".format(self.HOST)) sandbox = self.get_sandbox({}, {}, args, env, cls=sandbox_class(platform)) @@ -111,7 +113,7 @@ class TestTargetAndroid(TargetTest): def test_target(self): self.assertEqual( self.get_target(["--enable-project=mobile/android"]), - "arm-unknown-linux-androideabi", + "x86_64-unknown-linux-android", ) self.assertEqual( self.get_target(["--enable-project=mobile/android", "--target=i686"]), @@ -131,6 +133,16 @@ class TestTargetAndroid(TargetTest): ) +class TestTargetAndroidAppleSiliconHost(TargetTest): + HOST = "aarch64-apple-darwin" + + def test_target(self): + self.assertEqual( + self.get_target(["--enable-project=mobile/android"]), + "aarch64-unknown-linux-android", + ) + + class TestTargetOpenBSD(TargetTest): # config.guess returns amd64 on OpenBSD, which we need to pass through to # config.sub so that it canonicalizes to x86_64. diff --git a/python/mozbuild/mozbuild/vendor/moz_yaml.py b/python/mozbuild/mozbuild/vendor/moz_yaml.py index c094d22a2b..1e329d237c 100644 --- a/python/mozbuild/mozbuild/vendor/moz_yaml.py +++ b/python/mozbuild/mozbuild/vendor/moz_yaml.py @@ -57,7 +57,15 @@ VALID_LICENSES = [ "Unicode", # http://www.unicode.org/copyright.html ] -VALID_SOURCE_HOSTS = ["gitlab", "googlesource", "github", "angle", "codeberg", "git"] +VALID_SOURCE_HOSTS = [ + "gitlab", + "googlesource", + "github", + "angle", + "codeberg", + "git", + "yaml-dir", +] """ --- @@ -443,6 +451,7 @@ def _schema_1(): Length(min=1), In(VALID_SOURCE_HOSTS, msg="Unsupported Source Hosting"), ), + "source-host-path": str, "tracking": Match(r"^(commit|tag)$"), "release-artifact": All(str, Length(min=1)), "flavor": Match(r"^(regular|rust|individual-files)$"), diff --git a/python/mozbuild/mozbuild/vendor/vendor_manifest.py b/python/mozbuild/mozbuild/vendor/vendor_manifest.py index ad9564405e..7becdd4c2a 100644 --- a/python/mozbuild/mozbuild/vendor/vendor_manifest.py +++ b/python/mozbuild/mozbuild/vendor/vendor_manifest.py @@ -348,6 +348,23 @@ class VendorManifest(MozbuildObject): from mozbuild.vendor.host_codeberg import CodebergHost return CodebergHost(self.manifest) + elif self.manifest["vendoring"]["source-hosting"] == "yaml-dir": + import importlib.util + + modulename, classname = self.manifest["vendoring"][ + "source-host-path" + ].rsplit(".", 1) + spec = importlib.util.spec_from_file_location( + modulename, + os.path.join( + os.path.dirname(self.yaml_file), + modulename.replace(".", os.sep) + ".py", + ), + ) + module = importlib.util.module_from_spec(spec) + sys.modules[modulename] = module + spec.loader.exec_module(module) + return getattr(module, classname)(self.manifest) else: raise Exception( "Unknown source host: " + self.manifest["vendoring"]["source-hosting"] diff --git a/python/mozbuild/mozbuild/vendor/vendor_rust.py b/python/mozbuild/mozbuild/vendor/vendor_rust.py index 15b644cf99..d81f344f32 100644 --- a/python/mozbuild/mozbuild/vendor/vendor_rust.py +++ b/python/mozbuild/mozbuild/vendor/vendor_rust.py @@ -47,9 +47,9 @@ CARGO_CONFIG_TEMPLATE = """\ # Take advantage of the fact that cargo will treat lines starting with # # as comments to add preprocessing directives. This file can thus by copied -# as-is to $topsrcdir/.cargo/config with no preprocessing to be used there +# as-is to $topsrcdir/.cargo/config.toml with no preprocessing to be used there # (for e.g. independent tasks building rust code), or be preprocessed by -# the build system to produce a .cargo/config with the right content. +# the build system to produce a .cargo/config.toml with the right content. #define REPLACE_NAME {replace_name} #define VENDORED_DIRECTORY {directory} # We explicitly exclude the following section when preprocessing because @@ -275,6 +275,7 @@ Please commit or stash these changes before vendoring, or re-run with `--ignore- "ISC", "MIT", "MPL-2.0", + "Unicode-3.0", "Unicode-DFS-2016", "Unlicense", "Zlib", @@ -846,7 +847,7 @@ license file's hash. output = res.stdout.decode("UTF-8") # Get the snippet of configuration that cargo vendor outputs, and - # update .cargo/config with it. + # update .cargo/config.toml with it. # XXX(bug 1576765): Hopefully do something better after # https://github.com/rust-lang/cargo/issues/7280 is addressed. config = "\n".join( @@ -880,7 +881,7 @@ license file's hash. mozpath.normsep(os.path.normcase(self.topsrcdir)), ) - cargo_config = os.path.join(self.topsrcdir, ".cargo", "config.in") + cargo_config = os.path.join(self.topsrcdir, ".cargo", "config.toml.in") with open(cargo_config, "w", encoding="utf-8", newline="\n") as fh: fh.write( CARGO_CONFIG_TEMPLATE.format( diff --git a/python/mozbuild/mozpack/files.py b/python/mozbuild/mozpack/files.py index a320e1f4b4..9ae16d5c1c 100644 --- a/python/mozbuild/mozpack/files.py +++ b/python/mozbuild/mozpack/files.py @@ -901,7 +901,7 @@ class BaseFinder(object): if path.endswith((".ftl", ".properties")): return MinifiedCommentStripped(file) - if self._minify_js and path.endswith((".js", ".jsm")): + if self._minify_js and path.endswith((".js", ".jsm", ".mjs")): return MinifiedJavaScript(file, self._minify_js_verify_command) return file diff --git a/python/mozperftest/mozperftest/system/android_startup.py b/python/mozperftest/mozperftest/system/android_startup.py index abf5b04d55..349178f6d3 100644 --- a/python/mozperftest/mozperftest/system/android_startup.py +++ b/python/mozperftest/mozperftest/system/android_startup.py @@ -11,11 +11,13 @@ import mozdevice from .android import AndroidDevice DATETIME_FORMAT = "%Y.%m.%d" -PAGE_START = re.compile("GeckoSession: handleMessage GeckoView:PageStart uri=") +PAGE_START_MOZ = re.compile("GeckoSession: handleMessage GeckoView:PageStart uri=") PROD_FENIX = "fenix" PROD_FOCUS = "focus" -PROC_GVEX = "geckoview_example" +PROD_GVEX = "geckoview_example" +PROD_CHRM = "chrome-m" +MOZILLA_PRODUCTS = [PROD_FENIX, PROD_FOCUS, PROD_GVEX] KEY_NAME = "name" KEY_PRODUCT = "product" @@ -38,32 +40,44 @@ TEST_URI = "https://example.com" BASE_URL_DICT = { PROD_FENIX: ( "https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/" + "gecko.v2.mozilla-central.pushdate.{date}.latest.mobile.fenix-nightly" + "/artifacts/public%2Fbuild%2Ftarget.{architecture}.apk" + ), + PROD_FENIX + + "-pre-mono-repo": ( + "https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/" "mobile.v3.firefox-android.apks.fenix-nightly.{date}.latest.{architecture}/artifacts/" "public%2Fbuild%2Ftarget.{architecture}.apk" ), PROD_FENIX + "-latest": ( "https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/" - "mobile.v3.firefox-android.apks.fenix-nightly.latest.{architecture}/artifacts/" + "gecko.v2.mozilla-central.latest.mobile.fenix-nightly/artifacts/" "public%2Fbuild%2Ftarget.{architecture}.apk" ), PROD_FOCUS: ( "https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/" + "gecko.v2.mozilla-central.pushdate.{date}.latest.mobile.focus-nightly" + "/artifacts/public%2Fbuild%2Ftarget.{architecture}.apk" + ), + PROD_FOCUS + + "-pre-mono-repo": ( + "https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/" "mobile.v3.firefox-android.apks.focus-nightly.{date}.latest.{architecture}" "/artifacts/public%2Fbuild%2Ftarget.{architecture}.apk" ), PROD_FOCUS + "-latest": ( "https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/" - "mobile.v3.firefox-android.apks.focus-nightly.latest.{architecture}" - "/artifacts/public%2Fbuild%2Ftarget.{architecture}.apk" + "gecko.v2.mozilla-central.latest.mobile.focus-nightly/artifacts/" + "public%2Fbuild%2Ftarget.{architecture}.apk" ), - PROC_GVEX: ( + PROD_GVEX: ( "https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/" "gecko.v2.mozilla-central.pushdate.{date}.latest.mobile.android-" "{architecture}-debug/artifacts/public%2Fbuild%2Fgeckoview_example.apk" ), - PROC_GVEX + PROD_GVEX + "-latest": ( "https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/" "gecko.v2.mozilla-central.shippable.latest.mobile.android-" @@ -83,9 +97,12 @@ PROD_TO_CHANNEL_TO_PKGID = { "release": "org.mozilla.focus", "debug": "org.mozilla.focus.debug", }, - PROC_GVEX: { + PROD_GVEX: { "nightly": "org.mozilla.geckoview_example", }, + PROD_CHRM: { + "release": "com.android.chrome", + }, } TEST_LIST = [ "cold_main_first_frame", @@ -233,18 +250,18 @@ class AndroidStartUp(AndroidDevice): def run_performance_analysis(self, apk_metadata): # Installing the application on the device and getting ready to run the tests install_path = apk_metadata[KEY_NAME] + self.apk_name = apk_metadata[KEY_NAME].split(".")[0] if self.custom_apk_exists(): install_path = self.custom_apk_path - self.device.uninstall_app(self.package_id) - self.info(f"Installing {install_path}...") - app_name = self.device.install_app(install_path) - if self.device.is_app_installed(app_name): - self.info(f"Successfully installed {app_name}") - else: - raise AndroidStartUpInstallError("The android app was not installed") - self.apk_name = apk_metadata[KEY_NAME].split(".")[0] - + if self.product in MOZILLA_PRODUCTS: + self.device.uninstall_app(self.package_id) + self.info(f"Installing {install_path}...") + app_name = self.device.install_app(install_path) + if self.device.is_app_installed(app_name): + self.info(f"Successfully installed {app_name}") + else: + raise AndroidStartUpInstallError("The android app was not installed") return self.run_tests() def run_tests(self): @@ -285,12 +302,21 @@ class AndroidStartUp(AndroidDevice): def get_measurement(self, test_name, stdout): if test_name in [TEST_COLD_MAIN_FF, TEST_COLD_VIEW_FF]: return self.get_measurement_from_am_start_log(stdout) - elif test_name in [TEST_COLD_VIEW_NAV_START, TEST_COLD_MAIN_RESTORE]: + elif ( + test_name in [TEST_COLD_VIEW_NAV_START, TEST_COLD_MAIN_RESTORE] + and self.product in MOZILLA_PRODUCTS + ): # We must sleep until the Navigation::Start event occurs. If we don't # the script will fail. This can take up to 14s on the G5 time.sleep(17) proc = self.device.shell_output("logcat -d") return self.get_measurement_from_nav_start_logcat(proc) + else: + raise Exception( + "invalid test settings selected, please double check that " + "the test name is valid and that the test is supported for " + "the browser you are testing" + ) def get_measurement_from_am_start_log(self, stdout): total_time_prefix = "TotalTime: " @@ -321,7 +347,7 @@ class AndroidStartUp(AndroidDevice): return __line_to_datetime(proc_start_lines[0]) def __get_page_start_datetime(): - page_start_lines = [line for line in lines if PAGE_START.search(line)] + page_start_lines = [line for line in lines if PAGE_START_MOZ.search(line)] page_start_line_count = len(page_start_lines) page_start_assert_msg = "found len=" + str(page_start_line_count) @@ -405,10 +431,10 @@ class AndroidStartUp(AndroidDevice): TEST_COLD_MAIN_RESTORE, }: return - - # This sets mutable state so we only need to pass this flag once, before we start the test - self.device.shell( - f"am start-activity -W -a android.intent.action.MAIN --ez " - f"performancetest true -n{self.package_id}/org.mozilla.fenix.App" - ) - time.sleep(4) # ensure skip onboarding call has time to propagate. + if self.product == MOZILLA_PRODUCTS: + # This sets mutable state so we only need to pass this flag once, before we start the test + self.device.shell( + f"am start-activity -W -a android.intent.action.MAIN --ez " + f"performancetest true -n{self.package_id}/org.mozilla.fenix.App" + ) + time.sleep(4) # ensure skip onboarding call has time to propagate. diff --git a/python/mozperftest/mozperftest/test/webpagetest.py b/python/mozperftest/mozperftest/test/webpagetest.py index 82e62efa8c..8a9bac9f19 100644 --- a/python/mozperftest/mozperftest/test/webpagetest.py +++ b/python/mozperftest/mozperftest/test/webpagetest.py @@ -255,7 +255,7 @@ class WebPageTest(Layer): ) def request_with_timeout(self, url): - request_header = {"Host": "www.webpagetest.org"} + request_header = {"Host": "www.webpagetest.org", "X-WPT-API-KEY": self.WPT_key} requested_results = requests.get(url, headers=request_header) results_of_request = json.loads(requested_results.text) start = time.monotonic() @@ -320,7 +320,7 @@ class WebPageTest(Layer): for key_value_pair in list(options.items())[6:]: test_parameters += "&{}={}".format(*key_value_pair) return ( - f"https://webpagetest.org/runtest.php?url={website_to_be_tested}&k={self.WPT_key}&" + f"https://webpagetest.org/runtest.php?url={website_to_be_tested}&" f"location={options['location']}:{options['browser']}.{options['connection']}&" f"f=json{test_parameters}" ) diff --git a/python/mozperftest/mozperftest/tests/test_android_startup.py b/python/mozperftest/mozperftest/tests/test_android_startup.py index 9620a7d901..6bb24e5e0d 100644 --- a/python/mozperftest/mozperftest/tests/test_android_startup.py +++ b/python/mozperftest/mozperftest/tests/test_android_startup.py @@ -258,6 +258,7 @@ def test_custom_apk_startup(get_measurement_mock, time_sleep_mock, path_mock): test = android_startup.AndroidStartUp(env, mach_cmd) test.run_tests = lambda: True test.package_id = "FakeID" + test.product = "fenix" assert test.run_performance_analysis(SAMPLE_APK_METADATA) diff --git a/python/sites/docs.txt b/python/sites/docs.txt index 9bfc36ba2c..e72e0809fc 100644 --- a/python/sites/docs.txt +++ b/python/sites/docs.txt @@ -1,11 +1,11 @@ pth:tools/lint/eslint/ -pypi:Sphinx==6.2.1 +pypi:Sphinx==7.1.2 pypi:boto3==1.33.5 pypi:fluent.pygments==1.0 pypi:livereload==2.6.3 pypi:mots==0.10.0 pypi:myst-parser==2.0 -pypi:sphinx-copybutton==0.5.1 +pypi:sphinx-copybutton==0.5.2 pypi:sphinx-design==0.5.0 pypi:sphinx-js==3.2.2 pypi:sphinx-markdown-tables==0.0.17 diff --git a/python/sites/mach.txt b/python/sites/mach.txt index e1cb8f159c..90d6f153fc 100644 --- a/python/sites/mach.txt +++ b/python/sites/mach.txt @@ -93,7 +93,7 @@ vendored:third_party/python/wheel vendored:third_party/python/zipp # glean-sdk may not be installable if a wheel isn't available # and it has to be built from source. -pypi-optional:glean-sdk==58.1.0:telemetry will not be collected +pypi-optional:glean-sdk==59.0.0:telemetry will not be collected # Mach gracefully handles the case where `psutil` is unavailable. # We aren't (yet) able to pin packages in automation, so we have to # support down to the oldest locally-installed version (5.4.2). |