diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 01:13:27 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 01:13:27 +0000 |
commit | 40a355a42d4a9444dc753c04c6608dade2f06a23 (patch) | |
tree | 871fc667d2de662f171103ce5ec067014ef85e61 /python | |
parent | Adding upstream version 124.0.1. (diff) | |
download | firefox-40a355a42d4a9444dc753c04c6608dade2f06a23.tar.xz firefox-40a355a42d4a9444dc753c04c6608dade2f06a23.zip |
Adding upstream version 125.0.1.upstream/125.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'python')
44 files changed, 535 insertions, 119 deletions
diff --git a/python/l10n/fluent_migrations/bug_1845150_search_engine_notification.py b/python/l10n/fluent_migrations/bug_1845150_search_engine_notification.py index 125baa4c1f..a519833d9c 100644 --- a/python/l10n/fluent_migrations/bug_1845150_search_engine_notification.py +++ b/python/l10n/fluent_migrations/bug_1845150_search_engine_notification.py @@ -10,7 +10,7 @@ class STRIP_LABEL(TransformPattern): # Used to remove `<label data-l10n-name="remove-search-engine-article">` from a string def visit_TextElement(self, node): node.value = re.sub( - '\s?<label data-l10n-name="remove-search-engine-article">.+?</label>\s?', + r'\s?<label data-l10n-name="remove-search-engine-article">.+?</label>\s?', "", node.value, ) diff --git a/python/l10n/fluent_migrations/bug_1864340_autocomplete_footer.py b/python/l10n/fluent_migrations/bug_1864340_autocomplete_footer.py new file mode 100644 index 0000000000..e389f54861 --- /dev/null +++ b/python/l10n/fluent_migrations/bug_1864340_autocomplete_footer.py @@ -0,0 +1,59 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +import fluent.syntax.ast as FTL +from fluent.migrate.transforms import COPY, COPY_PATTERN + + +def migrate(ctx): + """Bug 1864340 - Convert autocomplete footer strings to FTL, part {index}.""" + + propertiesSource = "browser/extensions/formautofill/formautofill.properties" + fluentSource = "browser/browser/preferences/formAutofill.ftl" + target = "toolkit/toolkit/formautofill/formAutofill.ftl" + ctx.add_transforms( + target, + target, + [ + FTL.Message( + id=FTL.Identifier("autofill-manage-addresses-label"), + value=COPY(propertiesSource, "autocompleteManageAddresses"), + ), + FTL.Message( + id=FTL.Identifier("autofill-card-network-amex"), + value=COPY_PATTERN(fluentSource, "autofill-card-network-amex"), + ), + FTL.Message( + id=FTL.Identifier("autofill-card-network-cartebancaire"), + value=COPY_PATTERN(fluentSource, "autofill-card-network-cartebancaire"), + ), + FTL.Message( + id=FTL.Identifier("autofill-card-network-diners"), + value=COPY_PATTERN(fluentSource, "autofill-card-network-diners"), + ), + FTL.Message( + id=FTL.Identifier("autofill-card-network-discover"), + value=COPY_PATTERN(fluentSource, "autofill-card-network-discover"), + ), + FTL.Message( + id=FTL.Identifier("autofill-card-network-jcb"), + value=COPY_PATTERN(fluentSource, "autofill-card-network-jcb"), + ), + FTL.Message( + id=FTL.Identifier("autofill-card-network-mastercard"), + value=COPY_PATTERN(fluentSource, "autofill-card-network-mastercard"), + ), + FTL.Message( + id=FTL.Identifier("autofill-card-network-mir"), + value=COPY_PATTERN(fluentSource, "autofill-card-network-mir"), + ), + FTL.Message( + id=FTL.Identifier("autofill-card-network-unionpay"), + value=COPY_PATTERN(fluentSource, "autofill-card-network-unionpay"), + ), + FTL.Message( + id=FTL.Identifier("autofill-card-network-visa"), + value=COPY_PATTERN(fluentSource, "autofill-card-network-visa"), + ), + ], + ) diff --git a/python/l10n/fluent_migrations/bug_1864606_backlogged_crash_checkbox.py b/python/l10n/fluent_migrations/bug_1864606_backlogged_crash_checkbox.py new file mode 100644 index 0000000000..e224b1b6fa --- /dev/null +++ b/python/l10n/fluent_migrations/bug_1864606_backlogged_crash_checkbox.py @@ -0,0 +1,41 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +import re +import fluent.syntax.ast as FTL +from fluent.migrate.transforms import TransformPattern, COPY_PATTERN + + +class STRIP_ANCHOR(TransformPattern): + # Used to remove `<a data-l10n-name="crash-reports-link">[...]</a>` from a string + def visit_TextElement(self, node): + node.value = re.sub( + ' *<a data-l10n-name="crash-reports-link">.*</a>', "", node.value + ) + return node + + +def migrate(ctx): + """Bug 1864606 - Standardize the backlogged crash reports checkbox implementation, part {index}.""" + preferences_ftl = "browser/browser/preferences/preferences.ftl" + ctx.add_transforms( + preferences_ftl, + preferences_ftl, + [ + FTL.Message( + id=FTL.Identifier("collection-backlogged-crash-reports"), + attributes=[ + FTL.Attribute( + id=FTL.Identifier("accesskey"), + value=COPY_PATTERN( + preferences_ftl, + "collection-backlogged-crash-reports-with-link.accesskey", + ), + ), + ], + value=STRIP_ANCHOR( + preferences_ftl, "collection-backlogged-crash-reports-with-link" + ), + ), + ], + ) diff --git a/python/l10n/fluent_migrations/bug_1875957_web_appearance_warning.py b/python/l10n/fluent_migrations/bug_1875957_web_appearance_warning.py new file mode 100644 index 0000000000..b07bcdf36e --- /dev/null +++ b/python/l10n/fluent_migrations/bug_1875957_web_appearance_warning.py @@ -0,0 +1,40 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +import re +import fluent.syntax.ast as FTL +from fluent.migrate.transforms import TransformPattern + + +class STRIP_LINK(TransformPattern): + # Used to remove `<a data-l10n-name="colors-link">...</a>` from a string + def visit_TextElement(self, node): + node.value = re.sub( + '\s?<a data-l10n-name="colors-link">.+?</a>\s?', + "", + node.value, + ) + return node + + +def migrate(ctx): + """Bug 1875957 - Use moz-message-bar for web appearance warning, part {index}""" + preferences_ftl = "browser/browser/preferences/preferences.ftl" + ctx.add_transforms( + preferences_ftl, + preferences_ftl, + [ + FTL.Message( + id=FTL.Identifier("preferences-web-appearance-override-warning2"), + attributes=[ + FTL.Attribute( + id=FTL.Identifier("message"), + value=STRIP_LINK( + preferences_ftl, + "preferences-web-appearance-override-warning", + ), + ), + ], + ), + ], + ) diff --git a/python/mach/mach/command_util.py b/python/mach/mach/command_util.py index 741539f6f6..e8238bd83e 100644 --- a/python/mach/mach/command_util.py +++ b/python/mach/mach/command_util.py @@ -134,6 +134,7 @@ MACH_COMMANDS = { "mach-debug-commands": MachCommandReference( "python/mach/mach/commands/commandinfo.py" ), + "macos-sign": MachCommandReference("tools/signing/macos/mach_commands.py"), "manifest": MachCommandReference("testing/mach_commands.py"), "marionette-test": MachCommandReference("testing/marionette/mach_commands.py"), "mochitest": MachCommandReference("testing/mochitest/mach_commands.py", ["test"]), @@ -203,6 +204,7 @@ MACH_COMMANDS = { ), "tps-build": MachCommandReference("testing/tps/mach_commands.py"), "try": MachCommandReference("tools/tryselect/mach_commands.py"), + "ts": MachCommandReference("tools/ts/mach_commands.py"), "uniffi": MachCommandReference( "toolkit/components/uniffi-bindgen-gecko-js/mach_commands.py" ), diff --git a/python/mach/mach/util.py b/python/mach/mach/util.py index 203f08f92b..b6bf1727fa 100644 --- a/python/mach/mach/util.py +++ b/python/mach/mach/util.py @@ -115,3 +115,18 @@ def to_optional_str(path: Optional[Path]): return str(path) else: return None + + +def strtobool(value: str): + # Reimplementation of distutils.util.strtobool + # https://docs.python.org/3.9/distutils/apiref.html#distutils.util.strtobool + true_vals = ("y", "yes", "t", "true", "on", "1") + false_vals = ("n", "no", "f", "false", "off", "0") + + value = value.lower() + if value in true_vals: + return 1 + if value in false_vals: + return 0 + + raise ValueError(f'Expected one of: {", ".join(true_vals + false_vals)}') diff --git a/python/mozboot/bin/bootstrap.py b/python/mozboot/bin/bootstrap.py index 8009219c1d..9752705019 100755 --- a/python/mozboot/bin/bootstrap.py +++ b/python/mozboot/bin/bootstrap.py @@ -204,6 +204,8 @@ def git_clone_firefox(git: Path, dest: Path, watchman: Path, head_repo, head_rev subprocess.check_call( [ str(git), + "-c", + "fetch.prune=true", "clone", "--no-checkout", "hg::https://hg.mozilla.org/mozilla-unified", @@ -226,7 +228,12 @@ def git_clone_firefox(git: Path, dest: Path, watchman: Path, head_repo, head_rev ) subprocess.check_call( - [str(git), "checkout", "FETCH_HEAD" if head_rev else "bookmarks/central"], + [ + str(git), + "checkout", + "FETCH_HEAD" if head_rev else "bookmarks/central", + "--", + ], cwd=str(dest), env=env, ) diff --git a/python/mozboot/mozboot/android.py b/python/mozboot/mozboot/android.py index 116c5ff1ba..5c71339fa3 100644 --- a/python/mozboot/mozboot/android.py +++ b/python/mozboot/mozboot/android.py @@ -20,11 +20,11 @@ from tqdm import tqdm # variable. from mozboot.bootstrap import MOZCONFIG_SUGGESTION_TEMPLATE -NDK_VERSION = "r25c" -CMDLINE_TOOLS_VERSION_STRING = "11.0" -CMDLINE_TOOLS_VERSION = "9644228" +NDK_VERSION = "r26c" +CMDLINE_TOOLS_VERSION_STRING = "12.0" +CMDLINE_TOOLS_VERSION = "11076708" -BUNDLETOOL_VERSION = "1.15.2" +BUNDLETOOL_VERSION = "1.15.6" # We expect the emulator AVD definitions to be platform agnostic LINUX_X86_64_ANDROID_AVD = "linux64-android-avd-x86_64-repack" @@ -42,8 +42,8 @@ AVD_MANIFEST_ARM = Path(__file__).resolve().parent / "android-avds/arm.json" AVD_MANIFEST_ARM64 = Path(__file__).resolve().parent / "android-avds/arm64.json" JAVA_VERSION_MAJOR = "17" -JAVA_VERSION_MINOR = "0.9" -JAVA_VERSION_PATCH = "9" +JAVA_VERSION_MINOR = "0.10" +JAVA_VERSION_PATCH = "7" ANDROID_NDK_EXISTS = """ Looks like you have the correct version of the Android NDK installed at: @@ -824,41 +824,25 @@ def ensure_java(os_name, os_arch): if not java_path.exists(): # e.g. https://github.com/adoptium/temurin17-binaries/releases/ - # download/jdk-17.0.9%2B9/OpenJDK17U-jre_x64_linux_hotspot_17.0.9_9.tar.gz - if os_name != "windows": - java_url = ( - "https://github.com/adoptium/temurin{major}-binaries/releases/" - "download/jdk-{major}.{minor}%2B{patch}/" - "OpenJDK{major}U-jdk_{arch}_{os}_hotspot_{major}.{minor}_{patch}.{ext}" - ).format( - major=JAVA_VERSION_MAJOR, - minor=JAVA_VERSION_MINOR, - patch=JAVA_VERSION_PATCH, - os=os_tag, - arch=arch, - ext=ext, - ) - # Hack the URL for Windows due missed binary uploads for the original - # JDK 17.0.9 release. See bug 1870252. - else: - java_url = ( - "https://github.com/adoptium/temurin{major}-binaries/releases/" - "download/jdk-{major}.{minor}%2B{patch}.1/" - "OpenJDK{major}U-jdk_{arch}_{os}_hotspot_{major}.{minor}_{patch}.{ext}" - ).format( - major=JAVA_VERSION_MAJOR, - minor=JAVA_VERSION_MINOR, - patch=JAVA_VERSION_PATCH, - os=os_tag, - arch=arch, - ext=ext, - ) + # download/jdk-17.0.10%2B7/OpenJDK17U-jre_x64_linux_hotspot_17.0.10_7.tar.gz + java_url = ( + "https://github.com/adoptium/temurin{major}-binaries/releases/" + "download/jdk-{major}.{minor}%2B{patch}/" + "OpenJDK{major}U-jdk_{arch}_{os}_hotspot_{major}.{minor}_{patch}.{ext}" + ).format( + major=JAVA_VERSION_MAJOR, + minor=JAVA_VERSION_MINOR, + patch=JAVA_VERSION_PATCH, + os=os_tag, + arch=arch, + ext=ext, + ) install_mobile_android_sdk_or_ndk(java_url, mozbuild_path / "jdk") return java_path def java_bin_path(os_name, toolchain_path: Path): - # Like jdk-17.0.9+9 + # Like jdk-17.0.10+7 jdk_folder = "jdk-{major}.{minor}+{patch}".format( major=JAVA_VERSION_MAJOR, minor=JAVA_VERSION_MINOR, patch=JAVA_VERSION_PATCH ) diff --git a/python/mozboot/mozboot/debian.py b/python/mozboot/mozboot/debian.py index 63b47a2f03..9553800bf7 100644 --- a/python/mozboot/mozboot/debian.py +++ b/python/mozboot/mozboot/debian.py @@ -2,6 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +import subprocess import sys from mozboot.base import MERCURIAL_INSTALL_PROMPT, BaseBootstrapper @@ -60,7 +61,20 @@ class DebianBootstrapper(LinuxBootstrapper, BaseBootstrapper): assert res == 1 self.run_as_root(["pip3", "install", "--upgrade", "Mercurial"]) + def _check_packages_installed(self, *packages): + command = ["dpkg-query", "-W"] + command.extend(packages) + return ( + subprocess.run( + command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ).returncode + == 0 + ) + def apt_install(self, *packages): + if self._check_packages_installed(*packages): + # Packages already installed + return command = ["apt-get", "install"] if self.no_interactive: command.append("-y") diff --git a/python/mozboot/mozboot/util.py b/python/mozboot/mozboot/util.py index 47c35e670f..2d26ffb934 100644 --- a/python/mozboot/mozboot/util.py +++ b/python/mozboot/mozboot/util.py @@ -11,7 +11,7 @@ import certifi from mach.site import PythonVirtualenv from mach.util import get_state_dir -MINIMUM_RUST_VERSION = "1.70.0" +MINIMUM_RUST_VERSION = "1.74.0" def get_tools_dir(srcdir=False): diff --git a/python/mozbuild/mozbuild/action/langpack_manifest.py b/python/mozbuild/mozbuild/action/langpack_manifest.py index ffe32f567e..6995b12dc1 100644 --- a/python/mozbuild/mozbuild/action/langpack_manifest.py +++ b/python/mozbuild/mozbuild/action/langpack_manifest.py @@ -375,7 +375,7 @@ def parse_chrome_manifest(path, base_path, chrome_entries): ### def get_version_maybe_buildid(app_version): def _extract_numeric_part(part): - matches = re.compile("[^\d]").search(part) + matches = re.compile(r"[^\d]").search(part) if matches: part = part[0 : matches.start()] if len(part) == 0: diff --git a/python/mozbuild/mozbuild/action/xpidl-process.py b/python/mozbuild/mozbuild/action/xpidl-process.py index 0a126c729d..ca71c6543a 100755 --- a/python/mozbuild/mozbuild/action/xpidl-process.py +++ b/python/mozbuild/mozbuild/action/xpidl-process.py @@ -14,7 +14,7 @@ import sys import six from buildconfig import topsrcdir from mozpack import path as mozpath -from xpidl import jsonxpt +from xpidl import jsonxpt, typescript from xpidl.header import print_header from xpidl.rust import print_rust_bindings from xpidl.rust_macros import print_rust_macros_bindings @@ -39,6 +39,8 @@ def process( p = IDLParser() xpts = [] + ts_data = [] + mk = Makefile() rule = mk.create_rule() @@ -63,6 +65,7 @@ def process( rs_bt_path = os.path.join(xpcrs_dir, "bt", "%s.rs" % stem) xpts.append(jsonxpt.build_typelib(idl)) + ts_data.append(typescript.ts_source(idl)) rule.add_dependencies(six.ensure_text(s) for s in idl.deps) @@ -94,6 +97,13 @@ def process( with open(xpt_path, "w", encoding="utf-8", newline="\n") as fh: jsonxpt.write(jsonxpt.link(xpts), fh) + # NOTE: Make doesn't know about .d.json files, but we can piggy-back + # on XPT generation for now, as conceptually they contain the same + # information, and should be built together in all cases. + ts_path = os.path.join(xpt_dir, f"{module}.d.json") + with open(ts_path, "w", encoding="utf-8", newline="\n") as fh: + typescript.write(ts_data, fh) + rule.add_targets([six.ensure_text(xpt_path)]) if deps_dir: deps_path = os.path.join(deps_dir, "%s.pp" % module) diff --git a/python/mozbuild/mozbuild/artifacts.py b/python/mozbuild/mozbuild/artifacts.py index c82a39694c..a4d186816a 100644 --- a/python/mozbuild/mozbuild/artifacts.py +++ b/python/mozbuild/mozbuild/artifacts.py @@ -653,6 +653,7 @@ class MacArtifactJob(ArtifactJob): "{product}-bin", "*.dylib", "minidump-analyzer", + "nmhproxy", "pingsender", "plugin-container.app/Contents/MacOS/plugin-container", "updater.app/Contents/MacOS/org.mozilla.updater", diff --git a/python/mozbuild/mozbuild/backend/cpp_eclipse.py b/python/mozbuild/mozbuild/backend/cpp_eclipse.py index f2bd5ecd85..04735b702f 100644 --- a/python/mozbuild/mozbuild/backend/cpp_eclipse.py +++ b/python/mozbuild/mozbuild/backend/cpp_eclipse.py @@ -171,14 +171,14 @@ class CppEclipseBackend(CommonBackend): # Here we generate the code formatter that will show up in the UI with # the name "Mozilla". The formatter is stored as a single line of XML # in the org.eclipse.cdt.ui.formatterprofiles pref. - cdt_ui_prefs += """org.eclipse.cdt.ui.formatterprofiles=<?xml version\="1.0" encoding\="UTF-8" standalone\="no"?>\\n<profiles version\="1">\\n<profile kind\="CodeFormatterProfile" name\="Mozilla" version\="1">\\n""" - XML_PREF_TEMPLATE = """<setting id\="@PREF_NAME@" value\="@PREF_VAL@"/>\\n""" + cdt_ui_prefs += r'org.eclipse.cdt.ui.formatterprofiles=<?xml version\="1.0" encoding\="UTF-8" standalone\="no"?>\n<profiles version\="1">\n<profile kind\="CodeFormatterProfile" name\="Mozilla" version\="1">\n' + XML_PREF_TEMPLATE = r'<setting id\="@PREF_NAME@" value\="@PREF_VAL@"/>\n' for line in FORMATTER_SETTINGS.splitlines(): [pref, val] = line.split("=") cdt_ui_prefs += XML_PREF_TEMPLATE.replace("@PREF_NAME@", pref).replace( "@PREF_VAL@", val ) - cdt_ui_prefs += "</profile>\\n</profiles>\\n" + cdt_ui_prefs += r"</profile>\n</profiles>\n" with open(cdt_ui_prefs_path, "w") as fh: fh.write(cdt_ui_prefs) diff --git a/python/mozbuild/mozbuild/backend/mach_commands.py b/python/mozbuild/mozbuild/backend/mach_commands.py index 3feedefc42..8b0cdd5067 100644 --- a/python/mozbuild/mozbuild/backend/mach_commands.py +++ b/python/mozbuild/mozbuild/backend/mach_commands.py @@ -409,7 +409,7 @@ def get_clang_tools(command_context, clang_tools_path): def prompt_bool(prompt, limit=5): """Prompts the user with prompt and requires a boolean value.""" - from distutils.util import strtobool + from mach.util import strtobool for _ in range(limit): try: diff --git a/python/mozbuild/mozbuild/bootstrap.py b/python/mozbuild/mozbuild/bootstrap.py index 60a307145c..a21ad75ee4 100644 --- a/python/mozbuild/mozbuild/bootstrap.py +++ b/python/mozbuild/mozbuild/bootstrap.py @@ -37,9 +37,6 @@ def _bootstrap_sandbox(): Path(__file__).parent.parent.parent.parent / "build" / "moz.configure" ) sandbox.include_file(str(moz_configure / "init.configure")) - # bootstrap_search_path_order has a dependency on developer_options, which - # is not defined in init.configure. Its value doesn't matter for us, though. - sandbox["developer_options"] = sandbox["always"] sandbox.include_file(str(moz_configure / "bootstrap.configure")) return sandbox diff --git a/python/mozbuild/mozbuild/code_analysis/mach_commands.py b/python/mozbuild/mozbuild/code_analysis/mach_commands.py index a65d35c3cf..b0416679bb 100644 --- a/python/mozbuild/mozbuild/code_analysis/mach_commands.py +++ b/python/mozbuild/mozbuild/code_analysis/mach_commands.py @@ -51,7 +51,7 @@ def build_repo_relative_path(abs_path, repo_path): def prompt_bool(prompt, limit=5): """Prompts the user with prompt and requires a boolean value.""" - from distutils.util import strtobool + from mach.util import strtobool for _ in range(limit): try: diff --git a/python/mozbuild/mozbuild/codecoverage/chrome_map.py b/python/mozbuild/mozbuild/codecoverage/chrome_map.py index 79cedd2faf..ca21c9f62b 100644 --- a/python/mozbuild/mozbuild/codecoverage/chrome_map.py +++ b/python/mozbuild/mozbuild/codecoverage/chrome_map.py @@ -25,7 +25,7 @@ from mozbuild.frontend.data import ( from .manifest_handler import ChromeManifestHandler -_line_comment_re = re.compile('^//@line (\d+) "(.+)"$') +_line_comment_re = re.compile(r'^//@line (\d+) "(.+)"$') def generate_pp_info(path, topsrcdir): diff --git a/python/mozbuild/mozbuild/configure/__init__.py b/python/mozbuild/mozbuild/configure/__init__.py index 5a4edf3fb8..4c95fb08cb 100644 --- a/python/mozbuild/mozbuild/configure/__init__.py +++ b/python/mozbuild/mozbuild/configure/__init__.py @@ -49,6 +49,7 @@ class SandboxDependsFunction(object): def __init__(self, unsandboxed): self._or = unsandboxed.__or__ self._and = unsandboxed.__and__ + self._invert = unsandboxed.__invert__ self._getattr = unsandboxed.__getattr__ def __call__(self, *arg, **kwargs): @@ -70,6 +71,9 @@ class SandboxDependsFunction(object): ) return self._and(other).sandboxed + def __invert__(self): + return self._invert().sandboxed + def __cmp__(self, other): raise ConfigureError("Cannot compare @depends functions.") @@ -117,8 +121,6 @@ class DependsFunction(object): assert not inspect.isgeneratorfunction(func) # Allow non-functions when there are no dependencies. This is equivalent # to passing a lambda that returns the given value. - if not (inspect.isroutine(func) or not dependencies): - print(func) assert inspect.isroutine(func) or not dependencies self._func = func self._name = getattr(func, "__name__", None) @@ -190,6 +192,9 @@ class DependsFunction(object): assert self.sandbox is other.sandbox return CombinedDependsFunction(self.sandbox, self.and_impl, (self, other)) + def __invert__(self): + return TrivialDependsFunction(self.sandbox, lambda x: not x, [self]) + @staticmethod def and_impl(iterable): # Applies "and" to all the items of iterable. diff --git a/python/mozbuild/mozbuild/configure/check_debug_ranges.py b/python/mozbuild/mozbuild/configure/check_debug_ranges.py index f82624c14f..22c672ec9c 100644 --- a/python/mozbuild/mozbuild/configure/check_debug_ranges.py +++ b/python/mozbuild/mozbuild/configure/check_debug_ranges.py @@ -37,7 +37,7 @@ def get_range_length(range, debug_ranges): given offset.""" length = 0 for line in debug_ranges.splitlines(): - m = re.match("\s*([0-9a-fA-F]+)\s+([0-9a-fA-F]+)\s+([0-9a-fA-F]+)", line) + m = re.match(r"\s*([0-9a-fA-F]+)\s+([0-9a-fA-F]+)\s+([0-9a-fA-F]+)", line) if m and int(m.group(1), 16) == range: length += 1 return length diff --git a/python/mozbuild/mozbuild/configure/constants.py b/python/mozbuild/mozbuild/configure/constants.py index d69d9c08ef..25f43bb9f8 100644 --- a/python/mozbuild/mozbuild/configure/constants.py +++ b/python/mozbuild/mozbuild/configure/constants.py @@ -36,6 +36,7 @@ class OS(EnumString): "DragonFly", "FreeBSD", "GNU", + "iOS", "NetBSD", "OpenBSD", "OSX", diff --git a/python/mozbuild/mozbuild/dotproperties.py b/python/mozbuild/mozbuild/dotproperties.py index 9b615cc43f..c02e1a794b 100644 --- a/python/mozbuild/mozbuild/dotproperties.py +++ b/python/mozbuild/mozbuild/dotproperties.py @@ -38,7 +38,7 @@ class DotProperties: line = l.strip() if not line or line.startswith("#"): continue - (k, v) = re.split("\s*=\s*", line, 1) + (k, v) = re.split(r"\s*=\s*", line, 1) self._properties[k] = v def get(self, key, default=None): diff --git a/python/mozbuild/mozbuild/frontend/reader.py b/python/mozbuild/mozbuild/frontend/reader.py index d0066e1071..8c7dfcdcac 100644 --- a/python/mozbuild/mozbuild/frontend/reader.py +++ b/python/mozbuild/mozbuild/frontend/reader.py @@ -831,7 +831,7 @@ class BuildReader(object): self.config = config self._log = logging.getLogger(__name__) - self._read_files = set() + self._read_files = {} self._execution_stack = [] self.finder = finder @@ -1113,18 +1113,6 @@ class BuildReader(object): "Reading file: {path}".format(path=path), ) - if path in self._read_files: - log( - self._log, - logging.WARNING, - "read_already", - {"path": path}, - "File already read. Skipping: {path}".format(path=path), - ) - return - - self._read_files.add(path) - time_start = time.monotonic() topobjdir = config.topobjdir @@ -1132,6 +1120,20 @@ class BuildReader(object): relpath = mozpath.relpath(path, config.topsrcdir) reldir = mozpath.dirname(relpath) + # NOTE: descend case is handled in the loop below. + if not descend: + if relpath in self._read_files: + log( + self._log, + logging.WARNING, + "read_already", + {"path": path}, + "File already read. Skipping: {path}".format(path=path), + ) + return + + self._read_files[relpath] = (relpath, "") + if mozpath.dirname(relpath) == "js/src" and not config.substs.get( "JS_STANDALONE" ): @@ -1234,6 +1236,15 @@ class BuildReader(object): if not descend: continue + child_relpath = mozpath.relpath(child_path, self.config.topsrcdir) + + if child_relpath in self._read_files: + (prev_parent, prev_path) = self._read_files[child_relpath] + raise Exception( + f"File already read. A directory should not be added to DIRS twice: {child_relpath} is referred from {prev_parent} as '{prev_path}', and {relpath} as '{path}'" + ) + self._read_files[child_relpath] = (relpath, path) + for res in self.read_mozbuild( child_path, context.config, metadata=child_metadata ): diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py index 38723f4e0f..2398f8de03 100644 --- a/python/mozbuild/mozbuild/mach_commands.py +++ b/python/mozbuild/mozbuild/mach_commands.py @@ -2621,6 +2621,13 @@ def repackage_msi( help="Sign repackaged MSIX with self-signed certificate for local testing. " "(Default: false)", ) +@CommandArgument( + "--unsigned", + default=False, + action="store_true", + help="Support `Add-AppxPackage ... -AllowUnsigned` on Windows 11." + "(Default: false)", +) def repackage_msix( command_context, input, @@ -2636,6 +2643,7 @@ def repackage_msix( output=None, makeappx=None, sign=False, + unsigned=False, ): from mozbuild.repackaging.msix import repackage_msix @@ -2700,6 +2708,20 @@ def repackage_msix( ) return 1 + if unsigned: + if sign: + command_context.log( + logging.ERROR, + "repackage-msix-signed-and-unsigned", + {}, + "--sign and --unsigned are mutually exclusive", + ) + return 1 + + # Support `Add-AppxPackage ... -AllowUnsigned` on Windows 11. See + # https://github.com/MicrosoftDocs/msix-docs/blob/769dee9364df2b6fd0b78000774f8d14de8fe814/msix-src/package/unsigned-package.md. + publisher = f"{publisher}, OID.2.25.311729368913984317654407730594956997722=1" + output = repackage_msix( input, command_context.topsrcdir, diff --git a/python/mozbuild/mozbuild/preprocessor.py b/python/mozbuild/mozbuild/preprocessor.py index c81357efa4..b3a82a060c 100644 --- a/python/mozbuild/mozbuild/preprocessor.py +++ b/python/mozbuild/mozbuild/preprocessor.py @@ -29,7 +29,6 @@ import re import sys from optparse import OptionParser -import six from mozpack.path import normsep from mozbuild.makeutil import Makefile @@ -50,9 +49,11 @@ def _to_text(a): # We end up converting a lot of different types (text_type, binary_type, # int, etc.) to Unicode in this script. This function handles all of those # possibilities. - if isinstance(a, (six.text_type, six.binary_type)): - return six.ensure_text(a) - return six.text_type(a) + if isinstance(a, bytes): + return a.decode() + if isinstance(a, str): + return a + return str(a) def path_starts_with(path, prefix): @@ -540,9 +541,7 @@ class Preprocessor: self.processFile(input=input_, output=out) if depfile: mk = Makefile() - mk.create_rule([six.ensure_text(options.output)]).add_dependencies( - self.includes - ) + mk.create_rule([options.output]).add_dependencies(self.includes) mk.dump(depfile) depfile.close() @@ -712,7 +711,7 @@ class Preprocessor: except Exception: # XXX do real error reporting raise Preprocessor.Error(self, "SYNTAX_ERR", args) - if isinstance(val, six.text_type) or isinstance(val, six.binary_type): + if isinstance(val, (str, bytes)): # we're looking for a number value, strings are false val = False if not val: @@ -802,7 +801,7 @@ class Preprocessor: for i in range(1, len(lst), 2): lst[i] = vsubst(lst[i]) lst.append("\n") # add back the newline - self.write(six.moves.reduce(lambda x, y: x + y, lst, "")) + self.write("".join(lst)) def do_literal(self, args): self.write(args + "\n") @@ -863,7 +862,7 @@ class Preprocessor: args can either be a file name, or a file-like object. Files should be opened, and will be closed after processing. """ - isName = isinstance(args, six.string_types) + isName = isinstance(args, str) oldCheckLineNumbers = self.checkLineNumbers self.checkLineNumbers = False if isName: @@ -895,7 +894,7 @@ class Preprocessor: else: abspath = os.path.abspath(args.name) self.curdir = os.path.dirname(abspath) - self.includes.add(six.ensure_text(abspath)) + self.includes.add(abspath) if self.topobjdir and path_starts_with(abspath, self.topobjdir): abspath = "$OBJDIR" + normsep(abspath[len(self.topobjdir) :]) elif self.topsrcdir and path_starts_with(abspath, self.topsrcdir): diff --git a/python/mozbuild/mozbuild/repackaging/msix.py b/python/mozbuild/mozbuild/repackaging/msix.py index 762a33f1d1..0836ffb87a 100644 --- a/python/mozbuild/mozbuild/repackaging/msix.py +++ b/python/mozbuild/mozbuild/repackaging/msix.py @@ -813,7 +813,7 @@ def _sign_msix_win(output, force, log, verbose): thumbprint.strip() for thumbprint in powershell( ( - "Get-ChildItem -Path Cert:\CurrentUser\My" + r"Get-ChildItem -Path Cert:\CurrentUser\My" '| Where-Object {{$_.Subject -Match "{}"}}' '| Where-Object {{$_.FriendlyName -Match "{}"}}' "| Select-Object -ExpandProperty Thumbprint" @@ -838,7 +838,7 @@ def _sign_msix_win(output, force, log, verbose): ( 'New-SelfSignedCertificate -Type Custom -Subject "{}" ' '-KeyUsage DigitalSignature -FriendlyName "{}"' - " -CertStoreLocation Cert:\CurrentUser\My" + r" -CertStoreLocation Cert:\CurrentUser\My" ' -TextExtension @("2.5.29.37={{text}}1.3.6.1.5.5.7.3.3", ' '"2.5.29.19={{text}}")' "| Select-Object -ExpandProperty Thumbprint" @@ -856,7 +856,7 @@ def _sign_msix_win(output, force, log, verbose): ) powershell( - 'Export-Certificate -Cert Cert:\CurrentUser\My\{} -FilePath "{}"'.format( + r'Export-Certificate -Cert Cert:\CurrentUser\My\{} -FilePath "{}"'.format( thumbprint, crt_path ) ) @@ -869,7 +869,7 @@ def _sign_msix_win(output, force, log, verbose): powershell( ( - 'Export-PfxCertificate -Cert Cert:\CurrentUser\My\{} -FilePath "{}"' + r'Export-PfxCertificate -Cert Cert:\CurrentUser\My\{} -FilePath "{}"' ' -Password (ConvertTo-SecureString -String "{}" -Force -AsPlainText)' ).format(thumbprint, pfx_path, password) ) @@ -940,7 +940,7 @@ def _sign_msix_win(output, force, log, verbose): root_thumbprints = [ root_thumbprint.strip() for root_thumbprint in powershell( - "Get-ChildItem -Path Cert:\LocalMachine\Root\{} " + r"Get-ChildItem -Path Cert:\LocalMachine\Root\{} " "| Select-Object -ExpandProperty Thumbprint".format(thumbprint), check=False, ).splitlines() diff --git a/python/mozbuild/mozbuild/schedules.py b/python/mozbuild/mozbuild/schedules.py index 5f484ed377..0b7d9b1154 100644 --- a/python/mozbuild/mozbuild/schedules.py +++ b/python/mozbuild/mozbuild/schedules.py @@ -42,6 +42,7 @@ EXCLUSIVE_COMPONENTS = [ "linux", "macosx", "windows", + "ios", # broad test harness categories "awsy", "condprofile", diff --git a/python/mozbuild/mozbuild/test/configure/test_bootstrap.py b/python/mozbuild/mozbuild/test/configure/test_bootstrap.py index 758ddd5632..e3e3d3c744 100644 --- a/python/mozbuild/mozbuild/test/configure/test_bootstrap.py +++ b/python/mozbuild/mozbuild/test/configure/test_bootstrap.py @@ -37,7 +37,6 @@ class TestBootstrap(BaseConfigureTest): # - `in_path` is a 3-tuple representing whether the path for each toolchain is # expected to have been added to the `bootstrap_search_path`. Valid values are: # - `True`: the toolchain path was prepended to `bootstrap_search_path`. - # - `"append"`: the toolchain path was appended to `bootstrap_search_path`. # - `False`: the toolchain path is not in `bootstrap_search_path`. def assertBootstrap(self, arg, states, bootstrapped, in_path): called_for = [] @@ -119,7 +118,7 @@ class TestBootstrap(BaseConfigureTest): "--disable-bootstrap", (True, "old", False), (False, False, False), - (True, True, False), + (False, False, False), ) self.assertBootstrap( None, @@ -133,13 +132,13 @@ class TestBootstrap(BaseConfigureTest): "--disable-bootstrap", (True, "old", False), (False, False, False), - ("append", "append", False), + (False, False, False), ) self.assertBootstrap( None, (True, "old", False), (False, False, False), - ("append", "append", False), + (True, True, False), ) for milestone in ("124.0a1", "124.0"): @@ -151,6 +150,12 @@ class TestBootstrap(BaseConfigureTest): (False, True, True), (True, True, True), ) + self.assertBootstrap( + "--enable-bootstrap=no-update", + (True, "old", False), + (False, False, True), + (True, True, True), + ) # With `--enable-bootstrap=foo,bar`, only foo and bar are bootstrappable self.assertBootstrap( @@ -163,7 +168,19 @@ class TestBootstrap(BaseConfigureTest): "--enable-bootstrap=foo", (True, "old", True), (False, False, False), - (True, "append", "append"), + (True, True, True), + ) + self.assertBootstrap( + "--enable-bootstrap=no-update,foo,bar", + (False, "old", False), + (True, False, False), + (True, True, False), + ) + self.assertBootstrap( + "--enable-bootstrap=no-update,foo", + (True, "old", True), + (False, False, False), + (True, True, True), ) # With `--enable-bootstrap=-foo`, anything is bootstrappable, except foo @@ -171,7 +188,7 @@ class TestBootstrap(BaseConfigureTest): "--enable-bootstrap=-foo", (True, False, "old"), (False, True, True), - ("append", True, True), + (True, True, True), ) self.assertBootstrap( "--enable-bootstrap=-foo", @@ -179,13 +196,25 @@ class TestBootstrap(BaseConfigureTest): (False, True, True), (False, True, True), ) + self.assertBootstrap( + "--enable-bootstrap=no-update,-foo", + (True, False, "old"), + (False, True, False), + (True, True, True), + ) + self.assertBootstrap( + "--enable-bootstrap=no-update,-foo", + (False, False, "old"), + (False, True, False), + (False, True, True), + ) # Corner case. self.assertBootstrap( "--enable-bootstrap=-foo,foo,bar", (False, False, "old"), (False, True, False), - (False, True, "append"), + (False, True, True), ) diff --git a/python/mozbuild/mozbuild/test/configure/test_checks_configure.py b/python/mozbuild/mozbuild/test/configure/test_checks_configure.py index 3482f82f3d..131ac5aa7b 100644 --- a/python/mozbuild/mozbuild/test/configure/test_checks_configure.py +++ b/python/mozbuild/mozbuild/test/configure/test_checks_configure.py @@ -592,7 +592,7 @@ class TestChecksConfigure(unittest.TestCase): javac = mozpath.abspath("/usr/bin/javac") paths = {java: None, javac: None} expected_error_message = ( - "ERROR: Could not locate Java at /mozbuild/jdk/jdk-17.0.9+9/bin, " + "ERROR: Could not locate Java at /mozbuild/jdk/jdk-17.0.10+7/bin, " "please run ./mach bootstrap --no-system-changes\n" ) diff --git a/python/mozbuild/mozbuild/test/configure/test_configure.py b/python/mozbuild/mozbuild/test/configure/test_configure.py index d075477a44..a63ad9a15c 100644 --- a/python/mozbuild/mozbuild/test/configure/test_configure.py +++ b/python/mozbuild/mozbuild/test/configure/test_configure.py @@ -1921,6 +1921,72 @@ class TestConfigure(unittest.TestCase): ): self.get_config(["--enable-when"]) + def test_depends_unary_ops_func(self): + with self.moz_configure( + """ + option('--foo', nargs=1, help='foo') + @depends('--foo') + def foo(value): + return value + set_config('Foo', foo) + set_config('notFoo', depends(foo)(lambda x: not x)) + set_config('invFoo', ~foo) + """ + ): + foo_opt, foo_value = "--foo=foo", PositiveOptionValue(("foo",)) + + config = self.get_config([foo_opt]) + self.assertEqual( + config, + { + "Foo": foo_value, + "notFoo": not foo_value, + "invFoo": not foo_value, + }, + ) + + foo_value = False + config = self.get_config([]) + self.assertEqual( + config, + { + "Foo": foo_value, + "notFoo": not foo_value, + "invFoo": not foo_value, + }, + ) + + def test_depends_unary_ops_val(self): + with self.moz_configure( + """ + option("--cond", help="condition") + cond = depends("--cond")(lambda c: c) + foo = depends(when=cond)("foo") + set_config('Foo', foo) + set_config('notFoo', depends(foo)(lambda x: not x)) + set_config('invFoo', ~foo) + + bar = depends(when=~cond)("bar") + bar2 = depends(when=depends(cond)(lambda c: not c))("bar2") + set_config('Bar', bar) + set_config('Bar2', bar2) + """ + ): + config = self.get_config(["--cond"]) + self.assertEqual( + config, + { + "Foo": "foo", + "notFoo": not "foo", + "invFoo": not "foo", + }, + ) + config = self.get_config([]) + self.assertEqual( + config, + {"notFoo": True, "invFoo": True, "Bar": "bar", "Bar2": "bar2"}, + ) + def test_depends_binary_ops(self): with self.moz_configure( """ diff --git a/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py b/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py index f42778215b..f9e4840aa7 100644 --- a/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py +++ b/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py @@ -22,16 +22,16 @@ class CompilerPreprocessor(Preprocessor): # For now, we don't look very hard for C strings because they don't matter # that much for our unit tests, but we at least avoid expanding in the # simple "FOO" case. - VARSUBST = re.compile('(?<!")(?P<VAR>\w+)(?!")', re.U) - NON_WHITESPACE = re.compile("\S") + VARSUBST = re.compile(r'(?<!")(?P<VAR>\w+)(?!")', re.U) + NON_WHITESPACE = re.compile(r"\S") HAS_FEATURE_OR_BUILTIN = re.compile( - '(__has_(?:feature|builtin|attribute|warning))\("?([^"\)]*)"?\)' + r'(__has_(?:feature|builtin|attribute|warning))\("?([^"\)]*)"?\)' ) def __init__(self, *args, **kwargs): Preprocessor.__init__(self, *args, **kwargs) self.do_filter("c_substitution") - self.setMarker("#\s*") + self.setMarker(r"#\s*") def do_if(self, expression, **kwargs): # The C preprocessor handles numbers following C rules, which is a diff --git a/python/mozbuild/mozbuild/test/frontend/test_reader.py b/python/mozbuild/mozbuild/test/frontend/test_reader.py index a15bb15d7e..91d453a21f 100644 --- a/python/mozbuild/mozbuild/test/frontend/test_reader.py +++ b/python/mozbuild/mozbuild/test/frontend/test_reader.py @@ -83,12 +83,20 @@ class TestBuildReader(unittest.TestCase): contexts = list(reader.read_topsrcdir()) self.assertEqual(len(contexts), 3) - def test_repeated_dirs_ignored(self): - # Ensure repeated directories are ignored. + def test_repeated_dirs_error(self): reader = self.reader("traversal-repeated-dirs") - contexts = list(reader.read_topsrcdir()) - self.assertEqual(len(contexts), 3) + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertEqual( + e.actual_file, self.file_path("traversal-repeated-dirs", "bar", "moz.build") + ) + self.assertIn( + "File already read. A directory should not be added to DIRS twice: foo/moz.build is referred from moz.build as 'foo', and bar/moz.build as '../foo'", + str(e), + ) def test_outside_topsrcdir(self): # References to directories outside the topsrcdir should fail. diff --git a/python/mozbuild/mozbuild/vendor/rewrite_mozbuild.py b/python/mozbuild/mozbuild/vendor/rewrite_mozbuild.py index 8163c05dc3..5434510bb0 100644 --- a/python/mozbuild/mozbuild/vendor/rewrite_mozbuild.py +++ b/python/mozbuild/mozbuild/vendor/rewrite_mozbuild.py @@ -822,7 +822,7 @@ def edit_moz_build_file_to_remove_file( """ simple_file_line = re.compile( - "^\s*['\"]" + unnormalized_filename_to_remove + "['\"],*$" + "^\\s*['\"]" + unnormalized_filename_to_remove + "['\"],*$" ) did_replace = False diff --git a/python/mozbuild/mozbuild/vendor/vendor_manifest.py b/python/mozbuild/mozbuild/vendor/vendor_manifest.py index 65ee161348..ad9564405e 100644 --- a/python/mozbuild/mozbuild/vendor/vendor_manifest.py +++ b/python/mozbuild/mozbuild/vendor/vendor_manifest.py @@ -612,7 +612,7 @@ class VendorManifest(MozbuildObject): if r[0] in l: print("Found " + l) replaced += 1 - yaml[i] = re.sub(r[0] + " [v\.a-f0-9]+.*$", r[0] + r[1], yaml[i]) + yaml[i] = re.sub(r[0] + r" [v\.a-f0-9]+.*$", r[0] + r[1], yaml[i]) assert len(replacements) == replaced diff --git a/python/mozlint/mozlint/parser.py b/python/mozlint/mozlint/parser.py index eac502495b..213af88b4b 100644 --- a/python/mozlint/mozlint/parser.py +++ b/python/mozlint/mozlint/parser.py @@ -91,8 +91,15 @@ class Parser(object): "form 'module:object'".format(linter["setup"]), ) - if "extensions" in linter: - linter["extensions"] = [e.strip(".") for e in linter["extensions"]] + if "extensions" in linter and "exclude_extensions" in linter: + raise LinterParseError( + relpath, + "Can't have both 'extensions' and 'exclude_extensions'!", + ) + + for prop in ["extensions", "exclude_extensions"]: + if prop in linter: + linter[prop] = [e.strip(".") for e in linter[prop]] def parse(self, path): """Read a linter and return its LINTER definition. diff --git a/python/mozlint/mozlint/pathutils.py b/python/mozlint/mozlint/pathutils.py index 9b46fa6d41..eabfbafb8c 100644 --- a/python/mozlint/mozlint/pathutils.py +++ b/python/mozlint/mozlint/pathutils.py @@ -139,16 +139,20 @@ def collapse(paths, base=None, dotfiles=False): return list(covered) -def filterpaths(root, paths, include, exclude=None, extensions=None): +def filterpaths( + root, paths, include, exclude=None, extensions=None, exclude_extensions=None +): """Filters a list of paths. Given a list of paths and some filtering rules, return the set of paths - that should be linted. + that should be linted. Note that at most one of extensions or + exclude_extensions should be provided (ie not both). :param paths: A starting list of paths to possibly lint. :param include: A list of paths that should be included (required). :param exclude: A list of paths that should be excluded (optional). :param extensions: A list of file extensions which should be considered (optional). + :param exclude_extensions: A list of file extensions which should not be considered (optional). :returns: A tuple containing a list of file paths to lint and a list of paths to exclude. """ @@ -173,6 +177,8 @@ def filterpaths(root, paths, include, exclude=None, extensions=None): # Exclude bad file extensions if extensions and path.isfile and path.ext not in extensions: continue + elif exclude_extensions and path.isfile and path.ext in exclude_extensions: + continue if path.match(excludeglobs): continue @@ -280,6 +286,9 @@ def expand_exclusions(paths, config, root): Generator which generates list of paths that weren't excluded. """ extensions = [e.lstrip(".") for e in config.get("extensions", [])] + exclude_extensions = [e.lstrip(".") for e in config.get("exclude_extensions", [])] + if extensions and exclude_extensions: + raise ValueError("Can't specify both extensions and exclude_extensions.") find_dotfiles = config.get("find-dotfiles", False) def normalize(path): @@ -289,6 +298,13 @@ def expand_exclusions(paths, config, root): return mozpath.join(root, path) exclude = list(map(normalize, config.get("exclude", []))) + # We need excluded extensions in both the ignore for the FileFinder and in + # the exclusion set. If we don't put it in the exclusion set, we would + # return files that are passed explicitly and whose extensions are in the + # exclusion set. If we don't put it in the ignore set, the FileFinder + # would return files in (sub)directories passed to us. + base_ignore = ["**/*.{}".format(ext) for ext in exclude_extensions] + exclude += base_ignore for path in paths: path = mozpath.normsep(path) if os.path.isfile(path): @@ -301,16 +317,27 @@ def expand_exclusions(paths, config, root): yield path continue - ignore = [ + # If there are neither extensions nor exclude_extensions, we can't do + # anything useful with a directory. Skip: + if not extensions and not exclude_extensions: + continue + + # This is a directory. Check we don't have excludes for ancestors of + # this path. Mess with slashes to avoid "foo/bar" matching "foo/barry". + parent_path = os.path.dirname(path.rstrip("/")) + "/" + assert not any(parent_path.startswith(e.rstrip("/") + "/") for e in exclude) + + ignore = base_ignore + [ e[len(path) :].lstrip("/") for e in exclude if mozpath.commonprefix((path, e)) == path ] - finder = FileFinder(path, ignore=ignore, find_dotfiles=find_dotfiles) - - _, ext = os.path.splitext(path) - ext.lstrip(".") - for ext in extensions: - for p, f in finder.find("**/*.{}".format(ext)): + finder = FileFinder(path, ignore=ignore, find_dotfiles=find_dotfiles) + if extensions: + for ext in extensions: + for p, f in finder.find("**/*.{}".format(ext)): + yield os.path.join(path, p) + else: + for p, f in finder.find("**/*.*"): yield os.path.join(path, p) diff --git a/python/mozlint/mozlint/types.py b/python/mozlint/mozlint/types.py index 1a9a0bd473..468e28d81a 100644 --- a/python/mozlint/mozlint/types.py +++ b/python/mozlint/mozlint/types.py @@ -39,6 +39,7 @@ class BaseType(object): config["include"], config.get("exclude", []), config.get("extensions", []), + config.get("exclude_extensions", []), ) config["exclude"] = exclude elif config.get("exclude"): diff --git a/python/mozlint/test/linters/invalid_ext_and_exclude_ext.yml b/python/mozlint/test/linters/invalid_ext_and_exclude_ext.yml new file mode 100644 index 0000000000..a73353fb71 --- /dev/null +++ b/python/mozlint/test/linters/invalid_ext_and_exclude_ext.yml @@ -0,0 +1,7 @@ +--- +InvalidExtensionAndExcludedExtensions: + type: string + payload: foobar + description: Having both extensions and exclude_extensions is not allowed. + extensions: ["*.js"] + exclude_extensions: ["*.png"] diff --git a/python/mozlint/test/test_parser.py b/python/mozlint/test/test_parser.py index 2fbf26c8e5..3cb319a2e0 100644 --- a/python/mozlint/test/test_parser.py +++ b/python/mozlint/test/test_parser.py @@ -59,6 +59,7 @@ def test_parser_valid_multiple(parse): "invalid_include_with_glob.yml", "invalid_exclude.yml", "invalid_support_files.yml", + "invalid_ext_and_exclude_ext.yml", "missing_attrs.yml", "missing_definition.yml", "non_existing_include.yml", diff --git a/python/mozlint/test/test_pathutils.py b/python/mozlint/test/test_pathutils.py index 78f7883e88..5f593fdc47 100644 --- a/python/mozlint/test/test_pathutils.py +++ b/python/mozlint/test/test_pathutils.py @@ -59,6 +59,43 @@ def assert_paths(a, b): "extensions": ["py"], "expected": ["a.py", "subdir1/b.py"], }, + pytest.param( + { + "paths": [ + "a.py", + "a.js", + "subdir1/b.py", + "subdir2/c.py", + "subdir1/subdir3/d.py", + ], + "include": ["."], + "exclude": [], + "exclude_extensions": ["py"], + "expected": ["a.js"], + }, + id="Excluding .py should only return .js file.", + ), + pytest.param( + { + "paths": [ + "a.py", + "a.js", + "subdir1/b.py", + "subdir2/c.py", + "subdir1/subdir3/d.py", + ], + "include": ["."], + "exclude": [], + "exclude_extensions": ["js"], + "expected": [ + "a.py", + "subdir1/b.py", + "subdir2/c.py", + "subdir1/subdir3/d.py", + ], + }, + id="Excluding .js should only return .py files.", + ), { "paths": ["a.py", "a.js", "subdir2"], "include": ["."], @@ -104,24 +141,47 @@ def test_filterpaths(test): "paths": ["subdir1/b.js"], "config": { "exclude": ["subdir1"], - "extensions": "js", + "extensions": ["js"], }, "expected": [], }, { - "paths": ["subdir1/subdir3"], + "paths": ["subdir1"], "config": { "exclude": ["subdir1"], - "extensions": "js", + "extensions": ["js"], }, "expected": [], }, + pytest.param( + { + "paths": ["a.py", "subdir1"], + "config": { + "exclude": ["subdir1"], + "exclude_extensions": ["gob"], + }, + "expected": ["a.py"], + }, + id="Excluding both subdirs and nonsense extensions returns other files.", + ), + pytest.param( + { + "paths": ["a.py", "a.js", "subdir1"], + "config": { + "exclude": [], + "exclude_extensions": ["py"], + }, + "expected": ["a.js", "subdir1/subdir3/d.js", "subdir1/b.js"], + }, + id="Excluding .py files returns only non-.py files, also from subdirs.", + ), ), ) def test_expand_exclusions(test): expected = test.pop("expected", []) - paths = list(pathutils.expand_exclusions(test["paths"], test["config"], root)) + input_paths = [os.path.join(root, p) for p in test["paths"]] + paths = list(pathutils.expand_exclusions(input_paths, test["config"], root)) assert_paths(paths, expected) diff --git a/python/mozrelease/mozrelease/partner_repack.py b/python/mozrelease/mozrelease/partner_repack.py index 1d64f43cca..77b42a83c9 100644 --- a/python/mozrelease/mozrelease/partner_repack.py +++ b/python/mozrelease/mozrelease/partner_repack.py @@ -172,7 +172,7 @@ def parseRepackConfig(file: Path, platform: str): config[key] = value continue if key == "deb_section": - config["deb_section"] = re.sub("/", "\/", value) + config["deb_section"] = re.sub("/", r"\/", value) continue if isValidPlatform(key): ftp_platform = getFtpPlatform(key) diff --git a/python/sites/build.txt b/python/sites/build.txt index 07894139c0..18ed54c6a6 100644 --- a/python/sites/build.txt +++ b/python/sites/build.txt @@ -30,6 +30,7 @@ vendored:third_party/python/glean_parser vendored:third_party/python/gyp/pylib vendored:third_party/python/jinja2_time vendored:third_party/python/json_e +vendored:third_party/python/Mako vendored:third_party/python/mohawk vendored:third_party/python/mozilla_repo_urls vendored:third_party/python/multidict diff --git a/python/sites/docs.txt b/python/sites/docs.txt index 42413388a6..9bfc36ba2c 100644 --- a/python/sites/docs.txt +++ b/python/sites/docs.txt @@ -4,7 +4,7 @@ pypi:boto3==1.33.5 pypi:fluent.pygments==1.0 pypi:livereload==2.6.3 pypi:mots==0.10.0 -pypi:myst-parser==1.0 +pypi:myst-parser==2.0 pypi:sphinx-copybutton==0.5.1 pypi:sphinx-design==0.5.0 pypi:sphinx-js==3.2.2 diff --git a/python/sites/mach.txt b/python/sites/mach.txt index d9c98f24ce..e1cb8f159c 100644 --- a/python/sites/mach.txt +++ b/python/sites/mach.txt @@ -93,7 +93,7 @@ vendored:third_party/python/wheel vendored:third_party/python/zipp # glean-sdk may not be installable if a wheel isn't available # and it has to be built from source. -pypi-optional:glean-sdk==57.0.0:telemetry will not be collected +pypi-optional:glean-sdk==58.1.0:telemetry will not be collected # Mach gracefully handles the case where `psutil` is unavailable. # We aren't (yet) able to pin packages in automation, so we have to # support down to the oldest locally-installed version (5.4.2). |