summaryrefslogtreecommitdiffstats
path: root/python
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 01:13:33 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 01:13:33 +0000
commit086c044dc34dfc0f74fbe41f4ecb402b2cd34884 (patch)
treea4f824bd33cb075dd5aa3eb5a0a94af221bbe83a /python
parentAdding debian version 124.0.1-1. (diff)
downloadfirefox-086c044dc34dfc0f74fbe41f4ecb402b2cd34884.tar.xz
firefox-086c044dc34dfc0f74fbe41f4ecb402b2cd34884.zip
Merging upstream version 125.0.1.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'python')
-rw-r--r--python/l10n/fluent_migrations/bug_1845150_search_engine_notification.py2
-rw-r--r--python/l10n/fluent_migrations/bug_1864340_autocomplete_footer.py59
-rw-r--r--python/l10n/fluent_migrations/bug_1864606_backlogged_crash_checkbox.py41
-rw-r--r--python/l10n/fluent_migrations/bug_1875957_web_appearance_warning.py40
-rw-r--r--python/mach/mach/command_util.py2
-rw-r--r--python/mach/mach/util.py15
-rwxr-xr-xpython/mozboot/bin/bootstrap.py9
-rw-r--r--python/mozboot/mozboot/android.py56
-rw-r--r--python/mozboot/mozboot/debian.py14
-rw-r--r--python/mozboot/mozboot/util.py2
-rw-r--r--python/mozbuild/mozbuild/action/langpack_manifest.py2
-rwxr-xr-xpython/mozbuild/mozbuild/action/xpidl-process.py12
-rw-r--r--python/mozbuild/mozbuild/artifacts.py1
-rw-r--r--python/mozbuild/mozbuild/backend/cpp_eclipse.py6
-rw-r--r--python/mozbuild/mozbuild/backend/mach_commands.py2
-rw-r--r--python/mozbuild/mozbuild/bootstrap.py3
-rw-r--r--python/mozbuild/mozbuild/code_analysis/mach_commands.py2
-rw-r--r--python/mozbuild/mozbuild/codecoverage/chrome_map.py2
-rw-r--r--python/mozbuild/mozbuild/configure/__init__.py9
-rw-r--r--python/mozbuild/mozbuild/configure/check_debug_ranges.py2
-rw-r--r--python/mozbuild/mozbuild/configure/constants.py1
-rw-r--r--python/mozbuild/mozbuild/dotproperties.py2
-rw-r--r--python/mozbuild/mozbuild/frontend/reader.py37
-rw-r--r--python/mozbuild/mozbuild/mach_commands.py22
-rw-r--r--python/mozbuild/mozbuild/preprocessor.py21
-rw-r--r--python/mozbuild/mozbuild/repackaging/msix.py10
-rw-r--r--python/mozbuild/mozbuild/schedules.py1
-rw-r--r--python/mozbuild/mozbuild/test/configure/test_bootstrap.py43
-rw-r--r--python/mozbuild/mozbuild/test/configure/test_checks_configure.py2
-rw-r--r--python/mozbuild/mozbuild/test/configure/test_configure.py66
-rw-r--r--python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py8
-rw-r--r--python/mozbuild/mozbuild/test/frontend/test_reader.py16
-rw-r--r--python/mozbuild/mozbuild/vendor/rewrite_mozbuild.py2
-rw-r--r--python/mozbuild/mozbuild/vendor/vendor_manifest.py2
-rw-r--r--python/mozlint/mozlint/parser.py11
-rw-r--r--python/mozlint/mozlint/pathutils.py45
-rw-r--r--python/mozlint/mozlint/types.py1
-rw-r--r--python/mozlint/test/linters/invalid_ext_and_exclude_ext.yml7
-rw-r--r--python/mozlint/test/test_parser.py1
-rw-r--r--python/mozlint/test/test_pathutils.py68
-rw-r--r--python/mozrelease/mozrelease/partner_repack.py2
-rw-r--r--python/sites/build.txt1
-rw-r--r--python/sites/docs.txt2
-rw-r--r--python/sites/mach.txt2
44 files changed, 535 insertions, 119 deletions
diff --git a/python/l10n/fluent_migrations/bug_1845150_search_engine_notification.py b/python/l10n/fluent_migrations/bug_1845150_search_engine_notification.py
index 125baa4c1f..a519833d9c 100644
--- a/python/l10n/fluent_migrations/bug_1845150_search_engine_notification.py
+++ b/python/l10n/fluent_migrations/bug_1845150_search_engine_notification.py
@@ -10,7 +10,7 @@ class STRIP_LABEL(TransformPattern):
# Used to remove `<label data-l10n-name="remove-search-engine-article">` from a string
def visit_TextElement(self, node):
node.value = re.sub(
- '\s?<label data-l10n-name="remove-search-engine-article">.+?</label>\s?',
+ r'\s?<label data-l10n-name="remove-search-engine-article">.+?</label>\s?',
"",
node.value,
)
diff --git a/python/l10n/fluent_migrations/bug_1864340_autocomplete_footer.py b/python/l10n/fluent_migrations/bug_1864340_autocomplete_footer.py
new file mode 100644
index 0000000000..e389f54861
--- /dev/null
+++ b/python/l10n/fluent_migrations/bug_1864340_autocomplete_footer.py
@@ -0,0 +1,59 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+import fluent.syntax.ast as FTL
+from fluent.migrate.transforms import COPY, COPY_PATTERN
+
+
+def migrate(ctx):
+ """Bug 1864340 - Convert autocomplete footer strings to FTL, part {index}."""
+
+ propertiesSource = "browser/extensions/formautofill/formautofill.properties"
+ fluentSource = "browser/browser/preferences/formAutofill.ftl"
+ target = "toolkit/toolkit/formautofill/formAutofill.ftl"
+ ctx.add_transforms(
+ target,
+ target,
+ [
+ FTL.Message(
+ id=FTL.Identifier("autofill-manage-addresses-label"),
+ value=COPY(propertiesSource, "autocompleteManageAddresses"),
+ ),
+ FTL.Message(
+ id=FTL.Identifier("autofill-card-network-amex"),
+ value=COPY_PATTERN(fluentSource, "autofill-card-network-amex"),
+ ),
+ FTL.Message(
+ id=FTL.Identifier("autofill-card-network-cartebancaire"),
+ value=COPY_PATTERN(fluentSource, "autofill-card-network-cartebancaire"),
+ ),
+ FTL.Message(
+ id=FTL.Identifier("autofill-card-network-diners"),
+ value=COPY_PATTERN(fluentSource, "autofill-card-network-diners"),
+ ),
+ FTL.Message(
+ id=FTL.Identifier("autofill-card-network-discover"),
+ value=COPY_PATTERN(fluentSource, "autofill-card-network-discover"),
+ ),
+ FTL.Message(
+ id=FTL.Identifier("autofill-card-network-jcb"),
+ value=COPY_PATTERN(fluentSource, "autofill-card-network-jcb"),
+ ),
+ FTL.Message(
+ id=FTL.Identifier("autofill-card-network-mastercard"),
+ value=COPY_PATTERN(fluentSource, "autofill-card-network-mastercard"),
+ ),
+ FTL.Message(
+ id=FTL.Identifier("autofill-card-network-mir"),
+ value=COPY_PATTERN(fluentSource, "autofill-card-network-mir"),
+ ),
+ FTL.Message(
+ id=FTL.Identifier("autofill-card-network-unionpay"),
+ value=COPY_PATTERN(fluentSource, "autofill-card-network-unionpay"),
+ ),
+ FTL.Message(
+ id=FTL.Identifier("autofill-card-network-visa"),
+ value=COPY_PATTERN(fluentSource, "autofill-card-network-visa"),
+ ),
+ ],
+ )
diff --git a/python/l10n/fluent_migrations/bug_1864606_backlogged_crash_checkbox.py b/python/l10n/fluent_migrations/bug_1864606_backlogged_crash_checkbox.py
new file mode 100644
index 0000000000..e224b1b6fa
--- /dev/null
+++ b/python/l10n/fluent_migrations/bug_1864606_backlogged_crash_checkbox.py
@@ -0,0 +1,41 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+import re
+import fluent.syntax.ast as FTL
+from fluent.migrate.transforms import TransformPattern, COPY_PATTERN
+
+
+class STRIP_ANCHOR(TransformPattern):
+ # Used to remove `<a data-l10n-name="crash-reports-link">[...]</a>` from a string
+ def visit_TextElement(self, node):
+ node.value = re.sub(
+ ' *<a data-l10n-name="crash-reports-link">.*</a>', "", node.value
+ )
+ return node
+
+
+def migrate(ctx):
+ """Bug 1864606 - Standardize the backlogged crash reports checkbox implementation, part {index}."""
+ preferences_ftl = "browser/browser/preferences/preferences.ftl"
+ ctx.add_transforms(
+ preferences_ftl,
+ preferences_ftl,
+ [
+ FTL.Message(
+ id=FTL.Identifier("collection-backlogged-crash-reports"),
+ attributes=[
+ FTL.Attribute(
+ id=FTL.Identifier("accesskey"),
+ value=COPY_PATTERN(
+ preferences_ftl,
+ "collection-backlogged-crash-reports-with-link.accesskey",
+ ),
+ ),
+ ],
+ value=STRIP_ANCHOR(
+ preferences_ftl, "collection-backlogged-crash-reports-with-link"
+ ),
+ ),
+ ],
+ )
diff --git a/python/l10n/fluent_migrations/bug_1875957_web_appearance_warning.py b/python/l10n/fluent_migrations/bug_1875957_web_appearance_warning.py
new file mode 100644
index 0000000000..b07bcdf36e
--- /dev/null
+++ b/python/l10n/fluent_migrations/bug_1875957_web_appearance_warning.py
@@ -0,0 +1,40 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+import re
+import fluent.syntax.ast as FTL
+from fluent.migrate.transforms import TransformPattern
+
+
+class STRIP_LINK(TransformPattern):
+ # Used to remove `<a data-l10n-name="colors-link">...</a>` from a string
+ def visit_TextElement(self, node):
+ node.value = re.sub(
+ '\s?<a data-l10n-name="colors-link">.+?</a>\s?',
+ "",
+ node.value,
+ )
+ return node
+
+
+def migrate(ctx):
+ """Bug 1875957 - Use moz-message-bar for web appearance warning, part {index}"""
+ preferences_ftl = "browser/browser/preferences/preferences.ftl"
+ ctx.add_transforms(
+ preferences_ftl,
+ preferences_ftl,
+ [
+ FTL.Message(
+ id=FTL.Identifier("preferences-web-appearance-override-warning2"),
+ attributes=[
+ FTL.Attribute(
+ id=FTL.Identifier("message"),
+ value=STRIP_LINK(
+ preferences_ftl,
+ "preferences-web-appearance-override-warning",
+ ),
+ ),
+ ],
+ ),
+ ],
+ )
diff --git a/python/mach/mach/command_util.py b/python/mach/mach/command_util.py
index 741539f6f6..e8238bd83e 100644
--- a/python/mach/mach/command_util.py
+++ b/python/mach/mach/command_util.py
@@ -134,6 +134,7 @@ MACH_COMMANDS = {
"mach-debug-commands": MachCommandReference(
"python/mach/mach/commands/commandinfo.py"
),
+ "macos-sign": MachCommandReference("tools/signing/macos/mach_commands.py"),
"manifest": MachCommandReference("testing/mach_commands.py"),
"marionette-test": MachCommandReference("testing/marionette/mach_commands.py"),
"mochitest": MachCommandReference("testing/mochitest/mach_commands.py", ["test"]),
@@ -203,6 +204,7 @@ MACH_COMMANDS = {
),
"tps-build": MachCommandReference("testing/tps/mach_commands.py"),
"try": MachCommandReference("tools/tryselect/mach_commands.py"),
+ "ts": MachCommandReference("tools/ts/mach_commands.py"),
"uniffi": MachCommandReference(
"toolkit/components/uniffi-bindgen-gecko-js/mach_commands.py"
),
diff --git a/python/mach/mach/util.py b/python/mach/mach/util.py
index 203f08f92b..b6bf1727fa 100644
--- a/python/mach/mach/util.py
+++ b/python/mach/mach/util.py
@@ -115,3 +115,18 @@ def to_optional_str(path: Optional[Path]):
return str(path)
else:
return None
+
+
+def strtobool(value: str):
+ # Reimplementation of distutils.util.strtobool
+ # https://docs.python.org/3.9/distutils/apiref.html#distutils.util.strtobool
+ true_vals = ("y", "yes", "t", "true", "on", "1")
+ false_vals = ("n", "no", "f", "false", "off", "0")
+
+ value = value.lower()
+ if value in true_vals:
+ return 1
+ if value in false_vals:
+ return 0
+
+ raise ValueError(f'Expected one of: {", ".join(true_vals + false_vals)}')
diff --git a/python/mozboot/bin/bootstrap.py b/python/mozboot/bin/bootstrap.py
index 8009219c1d..9752705019 100755
--- a/python/mozboot/bin/bootstrap.py
+++ b/python/mozboot/bin/bootstrap.py
@@ -204,6 +204,8 @@ def git_clone_firefox(git: Path, dest: Path, watchman: Path, head_repo, head_rev
subprocess.check_call(
[
str(git),
+ "-c",
+ "fetch.prune=true",
"clone",
"--no-checkout",
"hg::https://hg.mozilla.org/mozilla-unified",
@@ -226,7 +228,12 @@ def git_clone_firefox(git: Path, dest: Path, watchman: Path, head_repo, head_rev
)
subprocess.check_call(
- [str(git), "checkout", "FETCH_HEAD" if head_rev else "bookmarks/central"],
+ [
+ str(git),
+ "checkout",
+ "FETCH_HEAD" if head_rev else "bookmarks/central",
+ "--",
+ ],
cwd=str(dest),
env=env,
)
diff --git a/python/mozboot/mozboot/android.py b/python/mozboot/mozboot/android.py
index 116c5ff1ba..5c71339fa3 100644
--- a/python/mozboot/mozboot/android.py
+++ b/python/mozboot/mozboot/android.py
@@ -20,11 +20,11 @@ from tqdm import tqdm
# variable.
from mozboot.bootstrap import MOZCONFIG_SUGGESTION_TEMPLATE
-NDK_VERSION = "r25c"
-CMDLINE_TOOLS_VERSION_STRING = "11.0"
-CMDLINE_TOOLS_VERSION = "9644228"
+NDK_VERSION = "r26c"
+CMDLINE_TOOLS_VERSION_STRING = "12.0"
+CMDLINE_TOOLS_VERSION = "11076708"
-BUNDLETOOL_VERSION = "1.15.2"
+BUNDLETOOL_VERSION = "1.15.6"
# We expect the emulator AVD definitions to be platform agnostic
LINUX_X86_64_ANDROID_AVD = "linux64-android-avd-x86_64-repack"
@@ -42,8 +42,8 @@ AVD_MANIFEST_ARM = Path(__file__).resolve().parent / "android-avds/arm.json"
AVD_MANIFEST_ARM64 = Path(__file__).resolve().parent / "android-avds/arm64.json"
JAVA_VERSION_MAJOR = "17"
-JAVA_VERSION_MINOR = "0.9"
-JAVA_VERSION_PATCH = "9"
+JAVA_VERSION_MINOR = "0.10"
+JAVA_VERSION_PATCH = "7"
ANDROID_NDK_EXISTS = """
Looks like you have the correct version of the Android NDK installed at:
@@ -824,41 +824,25 @@ def ensure_java(os_name, os_arch):
if not java_path.exists():
# e.g. https://github.com/adoptium/temurin17-binaries/releases/
- # download/jdk-17.0.9%2B9/OpenJDK17U-jre_x64_linux_hotspot_17.0.9_9.tar.gz
- if os_name != "windows":
- java_url = (
- "https://github.com/adoptium/temurin{major}-binaries/releases/"
- "download/jdk-{major}.{minor}%2B{patch}/"
- "OpenJDK{major}U-jdk_{arch}_{os}_hotspot_{major}.{minor}_{patch}.{ext}"
- ).format(
- major=JAVA_VERSION_MAJOR,
- minor=JAVA_VERSION_MINOR,
- patch=JAVA_VERSION_PATCH,
- os=os_tag,
- arch=arch,
- ext=ext,
- )
- # Hack the URL for Windows due missed binary uploads for the original
- # JDK 17.0.9 release. See bug 1870252.
- else:
- java_url = (
- "https://github.com/adoptium/temurin{major}-binaries/releases/"
- "download/jdk-{major}.{minor}%2B{patch}.1/"
- "OpenJDK{major}U-jdk_{arch}_{os}_hotspot_{major}.{minor}_{patch}.{ext}"
- ).format(
- major=JAVA_VERSION_MAJOR,
- minor=JAVA_VERSION_MINOR,
- patch=JAVA_VERSION_PATCH,
- os=os_tag,
- arch=arch,
- ext=ext,
- )
+ # download/jdk-17.0.10%2B7/OpenJDK17U-jre_x64_linux_hotspot_17.0.10_7.tar.gz
+ java_url = (
+ "https://github.com/adoptium/temurin{major}-binaries/releases/"
+ "download/jdk-{major}.{minor}%2B{patch}/"
+ "OpenJDK{major}U-jdk_{arch}_{os}_hotspot_{major}.{minor}_{patch}.{ext}"
+ ).format(
+ major=JAVA_VERSION_MAJOR,
+ minor=JAVA_VERSION_MINOR,
+ patch=JAVA_VERSION_PATCH,
+ os=os_tag,
+ arch=arch,
+ ext=ext,
+ )
install_mobile_android_sdk_or_ndk(java_url, mozbuild_path / "jdk")
return java_path
def java_bin_path(os_name, toolchain_path: Path):
- # Like jdk-17.0.9+9
+ # Like jdk-17.0.10+7
jdk_folder = "jdk-{major}.{minor}+{patch}".format(
major=JAVA_VERSION_MAJOR, minor=JAVA_VERSION_MINOR, patch=JAVA_VERSION_PATCH
)
diff --git a/python/mozboot/mozboot/debian.py b/python/mozboot/mozboot/debian.py
index 63b47a2f03..9553800bf7 100644
--- a/python/mozboot/mozboot/debian.py
+++ b/python/mozboot/mozboot/debian.py
@@ -2,6 +2,7 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import subprocess
import sys
from mozboot.base import MERCURIAL_INSTALL_PROMPT, BaseBootstrapper
@@ -60,7 +61,20 @@ class DebianBootstrapper(LinuxBootstrapper, BaseBootstrapper):
assert res == 1
self.run_as_root(["pip3", "install", "--upgrade", "Mercurial"])
+ def _check_packages_installed(self, *packages):
+ command = ["dpkg-query", "-W"]
+ command.extend(packages)
+ return (
+ subprocess.run(
+ command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+ ).returncode
+ == 0
+ )
+
def apt_install(self, *packages):
+ if self._check_packages_installed(*packages):
+ # Packages already installed
+ return
command = ["apt-get", "install"]
if self.no_interactive:
command.append("-y")
diff --git a/python/mozboot/mozboot/util.py b/python/mozboot/mozboot/util.py
index 47c35e670f..2d26ffb934 100644
--- a/python/mozboot/mozboot/util.py
+++ b/python/mozboot/mozboot/util.py
@@ -11,7 +11,7 @@ import certifi
from mach.site import PythonVirtualenv
from mach.util import get_state_dir
-MINIMUM_RUST_VERSION = "1.70.0"
+MINIMUM_RUST_VERSION = "1.74.0"
def get_tools_dir(srcdir=False):
diff --git a/python/mozbuild/mozbuild/action/langpack_manifest.py b/python/mozbuild/mozbuild/action/langpack_manifest.py
index ffe32f567e..6995b12dc1 100644
--- a/python/mozbuild/mozbuild/action/langpack_manifest.py
+++ b/python/mozbuild/mozbuild/action/langpack_manifest.py
@@ -375,7 +375,7 @@ def parse_chrome_manifest(path, base_path, chrome_entries):
###
def get_version_maybe_buildid(app_version):
def _extract_numeric_part(part):
- matches = re.compile("[^\d]").search(part)
+ matches = re.compile(r"[^\d]").search(part)
if matches:
part = part[0 : matches.start()]
if len(part) == 0:
diff --git a/python/mozbuild/mozbuild/action/xpidl-process.py b/python/mozbuild/mozbuild/action/xpidl-process.py
index 0a126c729d..ca71c6543a 100755
--- a/python/mozbuild/mozbuild/action/xpidl-process.py
+++ b/python/mozbuild/mozbuild/action/xpidl-process.py
@@ -14,7 +14,7 @@ import sys
import six
from buildconfig import topsrcdir
from mozpack import path as mozpath
-from xpidl import jsonxpt
+from xpidl import jsonxpt, typescript
from xpidl.header import print_header
from xpidl.rust import print_rust_bindings
from xpidl.rust_macros import print_rust_macros_bindings
@@ -39,6 +39,8 @@ def process(
p = IDLParser()
xpts = []
+ ts_data = []
+
mk = Makefile()
rule = mk.create_rule()
@@ -63,6 +65,7 @@ def process(
rs_bt_path = os.path.join(xpcrs_dir, "bt", "%s.rs" % stem)
xpts.append(jsonxpt.build_typelib(idl))
+ ts_data.append(typescript.ts_source(idl))
rule.add_dependencies(six.ensure_text(s) for s in idl.deps)
@@ -94,6 +97,13 @@ def process(
with open(xpt_path, "w", encoding="utf-8", newline="\n") as fh:
jsonxpt.write(jsonxpt.link(xpts), fh)
+ # NOTE: Make doesn't know about .d.json files, but we can piggy-back
+ # on XPT generation for now, as conceptually they contain the same
+ # information, and should be built together in all cases.
+ ts_path = os.path.join(xpt_dir, f"{module}.d.json")
+ with open(ts_path, "w", encoding="utf-8", newline="\n") as fh:
+ typescript.write(ts_data, fh)
+
rule.add_targets([six.ensure_text(xpt_path)])
if deps_dir:
deps_path = os.path.join(deps_dir, "%s.pp" % module)
diff --git a/python/mozbuild/mozbuild/artifacts.py b/python/mozbuild/mozbuild/artifacts.py
index c82a39694c..a4d186816a 100644
--- a/python/mozbuild/mozbuild/artifacts.py
+++ b/python/mozbuild/mozbuild/artifacts.py
@@ -653,6 +653,7 @@ class MacArtifactJob(ArtifactJob):
"{product}-bin",
"*.dylib",
"minidump-analyzer",
+ "nmhproxy",
"pingsender",
"plugin-container.app/Contents/MacOS/plugin-container",
"updater.app/Contents/MacOS/org.mozilla.updater",
diff --git a/python/mozbuild/mozbuild/backend/cpp_eclipse.py b/python/mozbuild/mozbuild/backend/cpp_eclipse.py
index f2bd5ecd85..04735b702f 100644
--- a/python/mozbuild/mozbuild/backend/cpp_eclipse.py
+++ b/python/mozbuild/mozbuild/backend/cpp_eclipse.py
@@ -171,14 +171,14 @@ class CppEclipseBackend(CommonBackend):
# Here we generate the code formatter that will show up in the UI with
# the name "Mozilla". The formatter is stored as a single line of XML
# in the org.eclipse.cdt.ui.formatterprofiles pref.
- cdt_ui_prefs += """org.eclipse.cdt.ui.formatterprofiles=<?xml version\="1.0" encoding\="UTF-8" standalone\="no"?>\\n<profiles version\="1">\\n<profile kind\="CodeFormatterProfile" name\="Mozilla" version\="1">\\n"""
- XML_PREF_TEMPLATE = """<setting id\="@PREF_NAME@" value\="@PREF_VAL@"/>\\n"""
+ cdt_ui_prefs += r'org.eclipse.cdt.ui.formatterprofiles=<?xml version\="1.0" encoding\="UTF-8" standalone\="no"?>\n<profiles version\="1">\n<profile kind\="CodeFormatterProfile" name\="Mozilla" version\="1">\n'
+ XML_PREF_TEMPLATE = r'<setting id\="@PREF_NAME@" value\="@PREF_VAL@"/>\n'
for line in FORMATTER_SETTINGS.splitlines():
[pref, val] = line.split("=")
cdt_ui_prefs += XML_PREF_TEMPLATE.replace("@PREF_NAME@", pref).replace(
"@PREF_VAL@", val
)
- cdt_ui_prefs += "</profile>\\n</profiles>\\n"
+ cdt_ui_prefs += r"</profile>\n</profiles>\n"
with open(cdt_ui_prefs_path, "w") as fh:
fh.write(cdt_ui_prefs)
diff --git a/python/mozbuild/mozbuild/backend/mach_commands.py b/python/mozbuild/mozbuild/backend/mach_commands.py
index 3feedefc42..8b0cdd5067 100644
--- a/python/mozbuild/mozbuild/backend/mach_commands.py
+++ b/python/mozbuild/mozbuild/backend/mach_commands.py
@@ -409,7 +409,7 @@ def get_clang_tools(command_context, clang_tools_path):
def prompt_bool(prompt, limit=5):
"""Prompts the user with prompt and requires a boolean value."""
- from distutils.util import strtobool
+ from mach.util import strtobool
for _ in range(limit):
try:
diff --git a/python/mozbuild/mozbuild/bootstrap.py b/python/mozbuild/mozbuild/bootstrap.py
index 60a307145c..a21ad75ee4 100644
--- a/python/mozbuild/mozbuild/bootstrap.py
+++ b/python/mozbuild/mozbuild/bootstrap.py
@@ -37,9 +37,6 @@ def _bootstrap_sandbox():
Path(__file__).parent.parent.parent.parent / "build" / "moz.configure"
)
sandbox.include_file(str(moz_configure / "init.configure"))
- # bootstrap_search_path_order has a dependency on developer_options, which
- # is not defined in init.configure. Its value doesn't matter for us, though.
- sandbox["developer_options"] = sandbox["always"]
sandbox.include_file(str(moz_configure / "bootstrap.configure"))
return sandbox
diff --git a/python/mozbuild/mozbuild/code_analysis/mach_commands.py b/python/mozbuild/mozbuild/code_analysis/mach_commands.py
index a65d35c3cf..b0416679bb 100644
--- a/python/mozbuild/mozbuild/code_analysis/mach_commands.py
+++ b/python/mozbuild/mozbuild/code_analysis/mach_commands.py
@@ -51,7 +51,7 @@ def build_repo_relative_path(abs_path, repo_path):
def prompt_bool(prompt, limit=5):
"""Prompts the user with prompt and requires a boolean value."""
- from distutils.util import strtobool
+ from mach.util import strtobool
for _ in range(limit):
try:
diff --git a/python/mozbuild/mozbuild/codecoverage/chrome_map.py b/python/mozbuild/mozbuild/codecoverage/chrome_map.py
index 79cedd2faf..ca21c9f62b 100644
--- a/python/mozbuild/mozbuild/codecoverage/chrome_map.py
+++ b/python/mozbuild/mozbuild/codecoverage/chrome_map.py
@@ -25,7 +25,7 @@ from mozbuild.frontend.data import (
from .manifest_handler import ChromeManifestHandler
-_line_comment_re = re.compile('^//@line (\d+) "(.+)"$')
+_line_comment_re = re.compile(r'^//@line (\d+) "(.+)"$')
def generate_pp_info(path, topsrcdir):
diff --git a/python/mozbuild/mozbuild/configure/__init__.py b/python/mozbuild/mozbuild/configure/__init__.py
index 5a4edf3fb8..4c95fb08cb 100644
--- a/python/mozbuild/mozbuild/configure/__init__.py
+++ b/python/mozbuild/mozbuild/configure/__init__.py
@@ -49,6 +49,7 @@ class SandboxDependsFunction(object):
def __init__(self, unsandboxed):
self._or = unsandboxed.__or__
self._and = unsandboxed.__and__
+ self._invert = unsandboxed.__invert__
self._getattr = unsandboxed.__getattr__
def __call__(self, *arg, **kwargs):
@@ -70,6 +71,9 @@ class SandboxDependsFunction(object):
)
return self._and(other).sandboxed
+ def __invert__(self):
+ return self._invert().sandboxed
+
def __cmp__(self, other):
raise ConfigureError("Cannot compare @depends functions.")
@@ -117,8 +121,6 @@ class DependsFunction(object):
assert not inspect.isgeneratorfunction(func)
# Allow non-functions when there are no dependencies. This is equivalent
# to passing a lambda that returns the given value.
- if not (inspect.isroutine(func) or not dependencies):
- print(func)
assert inspect.isroutine(func) or not dependencies
self._func = func
self._name = getattr(func, "__name__", None)
@@ -190,6 +192,9 @@ class DependsFunction(object):
assert self.sandbox is other.sandbox
return CombinedDependsFunction(self.sandbox, self.and_impl, (self, other))
+ def __invert__(self):
+ return TrivialDependsFunction(self.sandbox, lambda x: not x, [self])
+
@staticmethod
def and_impl(iterable):
# Applies "and" to all the items of iterable.
diff --git a/python/mozbuild/mozbuild/configure/check_debug_ranges.py b/python/mozbuild/mozbuild/configure/check_debug_ranges.py
index f82624c14f..22c672ec9c 100644
--- a/python/mozbuild/mozbuild/configure/check_debug_ranges.py
+++ b/python/mozbuild/mozbuild/configure/check_debug_ranges.py
@@ -37,7 +37,7 @@ def get_range_length(range, debug_ranges):
given offset."""
length = 0
for line in debug_ranges.splitlines():
- m = re.match("\s*([0-9a-fA-F]+)\s+([0-9a-fA-F]+)\s+([0-9a-fA-F]+)", line)
+ m = re.match(r"\s*([0-9a-fA-F]+)\s+([0-9a-fA-F]+)\s+([0-9a-fA-F]+)", line)
if m and int(m.group(1), 16) == range:
length += 1
return length
diff --git a/python/mozbuild/mozbuild/configure/constants.py b/python/mozbuild/mozbuild/configure/constants.py
index d69d9c08ef..25f43bb9f8 100644
--- a/python/mozbuild/mozbuild/configure/constants.py
+++ b/python/mozbuild/mozbuild/configure/constants.py
@@ -36,6 +36,7 @@ class OS(EnumString):
"DragonFly",
"FreeBSD",
"GNU",
+ "iOS",
"NetBSD",
"OpenBSD",
"OSX",
diff --git a/python/mozbuild/mozbuild/dotproperties.py b/python/mozbuild/mozbuild/dotproperties.py
index 9b615cc43f..c02e1a794b 100644
--- a/python/mozbuild/mozbuild/dotproperties.py
+++ b/python/mozbuild/mozbuild/dotproperties.py
@@ -38,7 +38,7 @@ class DotProperties:
line = l.strip()
if not line or line.startswith("#"):
continue
- (k, v) = re.split("\s*=\s*", line, 1)
+ (k, v) = re.split(r"\s*=\s*", line, 1)
self._properties[k] = v
def get(self, key, default=None):
diff --git a/python/mozbuild/mozbuild/frontend/reader.py b/python/mozbuild/mozbuild/frontend/reader.py
index d0066e1071..8c7dfcdcac 100644
--- a/python/mozbuild/mozbuild/frontend/reader.py
+++ b/python/mozbuild/mozbuild/frontend/reader.py
@@ -831,7 +831,7 @@ class BuildReader(object):
self.config = config
self._log = logging.getLogger(__name__)
- self._read_files = set()
+ self._read_files = {}
self._execution_stack = []
self.finder = finder
@@ -1113,18 +1113,6 @@ class BuildReader(object):
"Reading file: {path}".format(path=path),
)
- if path in self._read_files:
- log(
- self._log,
- logging.WARNING,
- "read_already",
- {"path": path},
- "File already read. Skipping: {path}".format(path=path),
- )
- return
-
- self._read_files.add(path)
-
time_start = time.monotonic()
topobjdir = config.topobjdir
@@ -1132,6 +1120,20 @@ class BuildReader(object):
relpath = mozpath.relpath(path, config.topsrcdir)
reldir = mozpath.dirname(relpath)
+ # NOTE: descend case is handled in the loop below.
+ if not descend:
+ if relpath in self._read_files:
+ log(
+ self._log,
+ logging.WARNING,
+ "read_already",
+ {"path": path},
+ "File already read. Skipping: {path}".format(path=path),
+ )
+ return
+
+ self._read_files[relpath] = (relpath, "")
+
if mozpath.dirname(relpath) == "js/src" and not config.substs.get(
"JS_STANDALONE"
):
@@ -1234,6 +1236,15 @@ class BuildReader(object):
if not descend:
continue
+ child_relpath = mozpath.relpath(child_path, self.config.topsrcdir)
+
+ if child_relpath in self._read_files:
+ (prev_parent, prev_path) = self._read_files[child_relpath]
+ raise Exception(
+ f"File already read. A directory should not be added to DIRS twice: {child_relpath} is referred from {prev_parent} as '{prev_path}', and {relpath} as '{path}'"
+ )
+ self._read_files[child_relpath] = (relpath, path)
+
for res in self.read_mozbuild(
child_path, context.config, metadata=child_metadata
):
diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py
index 38723f4e0f..2398f8de03 100644
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -2621,6 +2621,13 @@ def repackage_msi(
help="Sign repackaged MSIX with self-signed certificate for local testing. "
"(Default: false)",
)
+@CommandArgument(
+ "--unsigned",
+ default=False,
+ action="store_true",
+ help="Support `Add-AppxPackage ... -AllowUnsigned` on Windows 11."
+ "(Default: false)",
+)
def repackage_msix(
command_context,
input,
@@ -2636,6 +2643,7 @@ def repackage_msix(
output=None,
makeappx=None,
sign=False,
+ unsigned=False,
):
from mozbuild.repackaging.msix import repackage_msix
@@ -2700,6 +2708,20 @@ def repackage_msix(
)
return 1
+ if unsigned:
+ if sign:
+ command_context.log(
+ logging.ERROR,
+ "repackage-msix-signed-and-unsigned",
+ {},
+ "--sign and --unsigned are mutually exclusive",
+ )
+ return 1
+
+ # Support `Add-AppxPackage ... -AllowUnsigned` on Windows 11. See
+ # https://github.com/MicrosoftDocs/msix-docs/blob/769dee9364df2b6fd0b78000774f8d14de8fe814/msix-src/package/unsigned-package.md.
+ publisher = f"{publisher}, OID.2.25.311729368913984317654407730594956997722=1"
+
output = repackage_msix(
input,
command_context.topsrcdir,
diff --git a/python/mozbuild/mozbuild/preprocessor.py b/python/mozbuild/mozbuild/preprocessor.py
index c81357efa4..b3a82a060c 100644
--- a/python/mozbuild/mozbuild/preprocessor.py
+++ b/python/mozbuild/mozbuild/preprocessor.py
@@ -29,7 +29,6 @@ import re
import sys
from optparse import OptionParser
-import six
from mozpack.path import normsep
from mozbuild.makeutil import Makefile
@@ -50,9 +49,11 @@ def _to_text(a):
# We end up converting a lot of different types (text_type, binary_type,
# int, etc.) to Unicode in this script. This function handles all of those
# possibilities.
- if isinstance(a, (six.text_type, six.binary_type)):
- return six.ensure_text(a)
- return six.text_type(a)
+ if isinstance(a, bytes):
+ return a.decode()
+ if isinstance(a, str):
+ return a
+ return str(a)
def path_starts_with(path, prefix):
@@ -540,9 +541,7 @@ class Preprocessor:
self.processFile(input=input_, output=out)
if depfile:
mk = Makefile()
- mk.create_rule([six.ensure_text(options.output)]).add_dependencies(
- self.includes
- )
+ mk.create_rule([options.output]).add_dependencies(self.includes)
mk.dump(depfile)
depfile.close()
@@ -712,7 +711,7 @@ class Preprocessor:
except Exception:
# XXX do real error reporting
raise Preprocessor.Error(self, "SYNTAX_ERR", args)
- if isinstance(val, six.text_type) or isinstance(val, six.binary_type):
+ if isinstance(val, (str, bytes)):
# we're looking for a number value, strings are false
val = False
if not val:
@@ -802,7 +801,7 @@ class Preprocessor:
for i in range(1, len(lst), 2):
lst[i] = vsubst(lst[i])
lst.append("\n") # add back the newline
- self.write(six.moves.reduce(lambda x, y: x + y, lst, ""))
+ self.write("".join(lst))
def do_literal(self, args):
self.write(args + "\n")
@@ -863,7 +862,7 @@ class Preprocessor:
args can either be a file name, or a file-like object.
Files should be opened, and will be closed after processing.
"""
- isName = isinstance(args, six.string_types)
+ isName = isinstance(args, str)
oldCheckLineNumbers = self.checkLineNumbers
self.checkLineNumbers = False
if isName:
@@ -895,7 +894,7 @@ class Preprocessor:
else:
abspath = os.path.abspath(args.name)
self.curdir = os.path.dirname(abspath)
- self.includes.add(six.ensure_text(abspath))
+ self.includes.add(abspath)
if self.topobjdir and path_starts_with(abspath, self.topobjdir):
abspath = "$OBJDIR" + normsep(abspath[len(self.topobjdir) :])
elif self.topsrcdir and path_starts_with(abspath, self.topsrcdir):
diff --git a/python/mozbuild/mozbuild/repackaging/msix.py b/python/mozbuild/mozbuild/repackaging/msix.py
index 762a33f1d1..0836ffb87a 100644
--- a/python/mozbuild/mozbuild/repackaging/msix.py
+++ b/python/mozbuild/mozbuild/repackaging/msix.py
@@ -813,7 +813,7 @@ def _sign_msix_win(output, force, log, verbose):
thumbprint.strip()
for thumbprint in powershell(
(
- "Get-ChildItem -Path Cert:\CurrentUser\My"
+ r"Get-ChildItem -Path Cert:\CurrentUser\My"
'| Where-Object {{$_.Subject -Match "{}"}}'
'| Where-Object {{$_.FriendlyName -Match "{}"}}'
"| Select-Object -ExpandProperty Thumbprint"
@@ -838,7 +838,7 @@ def _sign_msix_win(output, force, log, verbose):
(
'New-SelfSignedCertificate -Type Custom -Subject "{}" '
'-KeyUsage DigitalSignature -FriendlyName "{}"'
- " -CertStoreLocation Cert:\CurrentUser\My"
+ r" -CertStoreLocation Cert:\CurrentUser\My"
' -TextExtension @("2.5.29.37={{text}}1.3.6.1.5.5.7.3.3", '
'"2.5.29.19={{text}}")'
"| Select-Object -ExpandProperty Thumbprint"
@@ -856,7 +856,7 @@ def _sign_msix_win(output, force, log, verbose):
)
powershell(
- 'Export-Certificate -Cert Cert:\CurrentUser\My\{} -FilePath "{}"'.format(
+ r'Export-Certificate -Cert Cert:\CurrentUser\My\{} -FilePath "{}"'.format(
thumbprint, crt_path
)
)
@@ -869,7 +869,7 @@ def _sign_msix_win(output, force, log, verbose):
powershell(
(
- 'Export-PfxCertificate -Cert Cert:\CurrentUser\My\{} -FilePath "{}"'
+ r'Export-PfxCertificate -Cert Cert:\CurrentUser\My\{} -FilePath "{}"'
' -Password (ConvertTo-SecureString -String "{}" -Force -AsPlainText)'
).format(thumbprint, pfx_path, password)
)
@@ -940,7 +940,7 @@ def _sign_msix_win(output, force, log, verbose):
root_thumbprints = [
root_thumbprint.strip()
for root_thumbprint in powershell(
- "Get-ChildItem -Path Cert:\LocalMachine\Root\{} "
+ r"Get-ChildItem -Path Cert:\LocalMachine\Root\{} "
"| Select-Object -ExpandProperty Thumbprint".format(thumbprint),
check=False,
).splitlines()
diff --git a/python/mozbuild/mozbuild/schedules.py b/python/mozbuild/mozbuild/schedules.py
index 5f484ed377..0b7d9b1154 100644
--- a/python/mozbuild/mozbuild/schedules.py
+++ b/python/mozbuild/mozbuild/schedules.py
@@ -42,6 +42,7 @@ EXCLUSIVE_COMPONENTS = [
"linux",
"macosx",
"windows",
+ "ios",
# broad test harness categories
"awsy",
"condprofile",
diff --git a/python/mozbuild/mozbuild/test/configure/test_bootstrap.py b/python/mozbuild/mozbuild/test/configure/test_bootstrap.py
index 758ddd5632..e3e3d3c744 100644
--- a/python/mozbuild/mozbuild/test/configure/test_bootstrap.py
+++ b/python/mozbuild/mozbuild/test/configure/test_bootstrap.py
@@ -37,7 +37,6 @@ class TestBootstrap(BaseConfigureTest):
# - `in_path` is a 3-tuple representing whether the path for each toolchain is
# expected to have been added to the `bootstrap_search_path`. Valid values are:
# - `True`: the toolchain path was prepended to `bootstrap_search_path`.
- # - `"append"`: the toolchain path was appended to `bootstrap_search_path`.
# - `False`: the toolchain path is not in `bootstrap_search_path`.
def assertBootstrap(self, arg, states, bootstrapped, in_path):
called_for = []
@@ -119,7 +118,7 @@ class TestBootstrap(BaseConfigureTest):
"--disable-bootstrap",
(True, "old", False),
(False, False, False),
- (True, True, False),
+ (False, False, False),
)
self.assertBootstrap(
None,
@@ -133,13 +132,13 @@ class TestBootstrap(BaseConfigureTest):
"--disable-bootstrap",
(True, "old", False),
(False, False, False),
- ("append", "append", False),
+ (False, False, False),
)
self.assertBootstrap(
None,
(True, "old", False),
(False, False, False),
- ("append", "append", False),
+ (True, True, False),
)
for milestone in ("124.0a1", "124.0"):
@@ -151,6 +150,12 @@ class TestBootstrap(BaseConfigureTest):
(False, True, True),
(True, True, True),
)
+ self.assertBootstrap(
+ "--enable-bootstrap=no-update",
+ (True, "old", False),
+ (False, False, True),
+ (True, True, True),
+ )
# With `--enable-bootstrap=foo,bar`, only foo and bar are bootstrappable
self.assertBootstrap(
@@ -163,7 +168,19 @@ class TestBootstrap(BaseConfigureTest):
"--enable-bootstrap=foo",
(True, "old", True),
(False, False, False),
- (True, "append", "append"),
+ (True, True, True),
+ )
+ self.assertBootstrap(
+ "--enable-bootstrap=no-update,foo,bar",
+ (False, "old", False),
+ (True, False, False),
+ (True, True, False),
+ )
+ self.assertBootstrap(
+ "--enable-bootstrap=no-update,foo",
+ (True, "old", True),
+ (False, False, False),
+ (True, True, True),
)
# With `--enable-bootstrap=-foo`, anything is bootstrappable, except foo
@@ -171,7 +188,7 @@ class TestBootstrap(BaseConfigureTest):
"--enable-bootstrap=-foo",
(True, False, "old"),
(False, True, True),
- ("append", True, True),
+ (True, True, True),
)
self.assertBootstrap(
"--enable-bootstrap=-foo",
@@ -179,13 +196,25 @@ class TestBootstrap(BaseConfigureTest):
(False, True, True),
(False, True, True),
)
+ self.assertBootstrap(
+ "--enable-bootstrap=no-update,-foo",
+ (True, False, "old"),
+ (False, True, False),
+ (True, True, True),
+ )
+ self.assertBootstrap(
+ "--enable-bootstrap=no-update,-foo",
+ (False, False, "old"),
+ (False, True, False),
+ (False, True, True),
+ )
# Corner case.
self.assertBootstrap(
"--enable-bootstrap=-foo,foo,bar",
(False, False, "old"),
(False, True, False),
- (False, True, "append"),
+ (False, True, True),
)
diff --git a/python/mozbuild/mozbuild/test/configure/test_checks_configure.py b/python/mozbuild/mozbuild/test/configure/test_checks_configure.py
index 3482f82f3d..131ac5aa7b 100644
--- a/python/mozbuild/mozbuild/test/configure/test_checks_configure.py
+++ b/python/mozbuild/mozbuild/test/configure/test_checks_configure.py
@@ -592,7 +592,7 @@ class TestChecksConfigure(unittest.TestCase):
javac = mozpath.abspath("/usr/bin/javac")
paths = {java: None, javac: None}
expected_error_message = (
- "ERROR: Could not locate Java at /mozbuild/jdk/jdk-17.0.9+9/bin, "
+ "ERROR: Could not locate Java at /mozbuild/jdk/jdk-17.0.10+7/bin, "
"please run ./mach bootstrap --no-system-changes\n"
)
diff --git a/python/mozbuild/mozbuild/test/configure/test_configure.py b/python/mozbuild/mozbuild/test/configure/test_configure.py
index d075477a44..a63ad9a15c 100644
--- a/python/mozbuild/mozbuild/test/configure/test_configure.py
+++ b/python/mozbuild/mozbuild/test/configure/test_configure.py
@@ -1921,6 +1921,72 @@ class TestConfigure(unittest.TestCase):
):
self.get_config(["--enable-when"])
+ def test_depends_unary_ops_func(self):
+ with self.moz_configure(
+ """
+ option('--foo', nargs=1, help='foo')
+ @depends('--foo')
+ def foo(value):
+ return value
+ set_config('Foo', foo)
+ set_config('notFoo', depends(foo)(lambda x: not x))
+ set_config('invFoo', ~foo)
+ """
+ ):
+ foo_opt, foo_value = "--foo=foo", PositiveOptionValue(("foo",))
+
+ config = self.get_config([foo_opt])
+ self.assertEqual(
+ config,
+ {
+ "Foo": foo_value,
+ "notFoo": not foo_value,
+ "invFoo": not foo_value,
+ },
+ )
+
+ foo_value = False
+ config = self.get_config([])
+ self.assertEqual(
+ config,
+ {
+ "Foo": foo_value,
+ "notFoo": not foo_value,
+ "invFoo": not foo_value,
+ },
+ )
+
+ def test_depends_unary_ops_val(self):
+ with self.moz_configure(
+ """
+ option("--cond", help="condition")
+ cond = depends("--cond")(lambda c: c)
+ foo = depends(when=cond)("foo")
+ set_config('Foo', foo)
+ set_config('notFoo', depends(foo)(lambda x: not x))
+ set_config('invFoo', ~foo)
+
+ bar = depends(when=~cond)("bar")
+ bar2 = depends(when=depends(cond)(lambda c: not c))("bar2")
+ set_config('Bar', bar)
+ set_config('Bar2', bar2)
+ """
+ ):
+ config = self.get_config(["--cond"])
+ self.assertEqual(
+ config,
+ {
+ "Foo": "foo",
+ "notFoo": not "foo",
+ "invFoo": not "foo",
+ },
+ )
+ config = self.get_config([])
+ self.assertEqual(
+ config,
+ {"notFoo": True, "invFoo": True, "Bar": "bar", "Bar2": "bar2"},
+ )
+
def test_depends_binary_ops(self):
with self.moz_configure(
"""
diff --git a/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py b/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py
index f42778215b..f9e4840aa7 100644
--- a/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py
+++ b/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py
@@ -22,16 +22,16 @@ class CompilerPreprocessor(Preprocessor):
# For now, we don't look very hard for C strings because they don't matter
# that much for our unit tests, but we at least avoid expanding in the
# simple "FOO" case.
- VARSUBST = re.compile('(?<!")(?P<VAR>\w+)(?!")', re.U)
- NON_WHITESPACE = re.compile("\S")
+ VARSUBST = re.compile(r'(?<!")(?P<VAR>\w+)(?!")', re.U)
+ NON_WHITESPACE = re.compile(r"\S")
HAS_FEATURE_OR_BUILTIN = re.compile(
- '(__has_(?:feature|builtin|attribute|warning))\("?([^"\)]*)"?\)'
+ r'(__has_(?:feature|builtin|attribute|warning))\("?([^"\)]*)"?\)'
)
def __init__(self, *args, **kwargs):
Preprocessor.__init__(self, *args, **kwargs)
self.do_filter("c_substitution")
- self.setMarker("#\s*")
+ self.setMarker(r"#\s*")
def do_if(self, expression, **kwargs):
# The C preprocessor handles numbers following C rules, which is a
diff --git a/python/mozbuild/mozbuild/test/frontend/test_reader.py b/python/mozbuild/mozbuild/test/frontend/test_reader.py
index a15bb15d7e..91d453a21f 100644
--- a/python/mozbuild/mozbuild/test/frontend/test_reader.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_reader.py
@@ -83,12 +83,20 @@ class TestBuildReader(unittest.TestCase):
contexts = list(reader.read_topsrcdir())
self.assertEqual(len(contexts), 3)
- def test_repeated_dirs_ignored(self):
- # Ensure repeated directories are ignored.
+ def test_repeated_dirs_error(self):
reader = self.reader("traversal-repeated-dirs")
- contexts = list(reader.read_topsrcdir())
- self.assertEqual(len(contexts), 3)
+ with self.assertRaises(BuildReaderError) as bre:
+ list(reader.read_topsrcdir())
+
+ e = bre.exception
+ self.assertEqual(
+ e.actual_file, self.file_path("traversal-repeated-dirs", "bar", "moz.build")
+ )
+ self.assertIn(
+ "File already read. A directory should not be added to DIRS twice: foo/moz.build is referred from moz.build as 'foo', and bar/moz.build as '../foo'",
+ str(e),
+ )
def test_outside_topsrcdir(self):
# References to directories outside the topsrcdir should fail.
diff --git a/python/mozbuild/mozbuild/vendor/rewrite_mozbuild.py b/python/mozbuild/mozbuild/vendor/rewrite_mozbuild.py
index 8163c05dc3..5434510bb0 100644
--- a/python/mozbuild/mozbuild/vendor/rewrite_mozbuild.py
+++ b/python/mozbuild/mozbuild/vendor/rewrite_mozbuild.py
@@ -822,7 +822,7 @@ def edit_moz_build_file_to_remove_file(
"""
simple_file_line = re.compile(
- "^\s*['\"]" + unnormalized_filename_to_remove + "['\"],*$"
+ "^\\s*['\"]" + unnormalized_filename_to_remove + "['\"],*$"
)
did_replace = False
diff --git a/python/mozbuild/mozbuild/vendor/vendor_manifest.py b/python/mozbuild/mozbuild/vendor/vendor_manifest.py
index 65ee161348..ad9564405e 100644
--- a/python/mozbuild/mozbuild/vendor/vendor_manifest.py
+++ b/python/mozbuild/mozbuild/vendor/vendor_manifest.py
@@ -612,7 +612,7 @@ class VendorManifest(MozbuildObject):
if r[0] in l:
print("Found " + l)
replaced += 1
- yaml[i] = re.sub(r[0] + " [v\.a-f0-9]+.*$", r[0] + r[1], yaml[i])
+ yaml[i] = re.sub(r[0] + r" [v\.a-f0-9]+.*$", r[0] + r[1], yaml[i])
assert len(replacements) == replaced
diff --git a/python/mozlint/mozlint/parser.py b/python/mozlint/mozlint/parser.py
index eac502495b..213af88b4b 100644
--- a/python/mozlint/mozlint/parser.py
+++ b/python/mozlint/mozlint/parser.py
@@ -91,8 +91,15 @@ class Parser(object):
"form 'module:object'".format(linter["setup"]),
)
- if "extensions" in linter:
- linter["extensions"] = [e.strip(".") for e in linter["extensions"]]
+ if "extensions" in linter and "exclude_extensions" in linter:
+ raise LinterParseError(
+ relpath,
+ "Can't have both 'extensions' and 'exclude_extensions'!",
+ )
+
+ for prop in ["extensions", "exclude_extensions"]:
+ if prop in linter:
+ linter[prop] = [e.strip(".") for e in linter[prop]]
def parse(self, path):
"""Read a linter and return its LINTER definition.
diff --git a/python/mozlint/mozlint/pathutils.py b/python/mozlint/mozlint/pathutils.py
index 9b46fa6d41..eabfbafb8c 100644
--- a/python/mozlint/mozlint/pathutils.py
+++ b/python/mozlint/mozlint/pathutils.py
@@ -139,16 +139,20 @@ def collapse(paths, base=None, dotfiles=False):
return list(covered)
-def filterpaths(root, paths, include, exclude=None, extensions=None):
+def filterpaths(
+ root, paths, include, exclude=None, extensions=None, exclude_extensions=None
+):
"""Filters a list of paths.
Given a list of paths and some filtering rules, return the set of paths
- that should be linted.
+ that should be linted. Note that at most one of extensions or
+ exclude_extensions should be provided (ie not both).
:param paths: A starting list of paths to possibly lint.
:param include: A list of paths that should be included (required).
:param exclude: A list of paths that should be excluded (optional).
:param extensions: A list of file extensions which should be considered (optional).
+ :param exclude_extensions: A list of file extensions which should not be considered (optional).
:returns: A tuple containing a list of file paths to lint and a list of
paths to exclude.
"""
@@ -173,6 +177,8 @@ def filterpaths(root, paths, include, exclude=None, extensions=None):
# Exclude bad file extensions
if extensions and path.isfile and path.ext not in extensions:
continue
+ elif exclude_extensions and path.isfile and path.ext in exclude_extensions:
+ continue
if path.match(excludeglobs):
continue
@@ -280,6 +286,9 @@ def expand_exclusions(paths, config, root):
Generator which generates list of paths that weren't excluded.
"""
extensions = [e.lstrip(".") for e in config.get("extensions", [])]
+ exclude_extensions = [e.lstrip(".") for e in config.get("exclude_extensions", [])]
+ if extensions and exclude_extensions:
+ raise ValueError("Can't specify both extensions and exclude_extensions.")
find_dotfiles = config.get("find-dotfiles", False)
def normalize(path):
@@ -289,6 +298,13 @@ def expand_exclusions(paths, config, root):
return mozpath.join(root, path)
exclude = list(map(normalize, config.get("exclude", [])))
+ # We need excluded extensions in both the ignore for the FileFinder and in
+ # the exclusion set. If we don't put it in the exclusion set, we would
+ # return files that are passed explicitly and whose extensions are in the
+ # exclusion set. If we don't put it in the ignore set, the FileFinder
+ # would return files in (sub)directories passed to us.
+ base_ignore = ["**/*.{}".format(ext) for ext in exclude_extensions]
+ exclude += base_ignore
for path in paths:
path = mozpath.normsep(path)
if os.path.isfile(path):
@@ -301,16 +317,27 @@ def expand_exclusions(paths, config, root):
yield path
continue
- ignore = [
+ # If there are neither extensions nor exclude_extensions, we can't do
+ # anything useful with a directory. Skip:
+ if not extensions and not exclude_extensions:
+ continue
+
+ # This is a directory. Check we don't have excludes for ancestors of
+ # this path. Mess with slashes to avoid "foo/bar" matching "foo/barry".
+ parent_path = os.path.dirname(path.rstrip("/")) + "/"
+ assert not any(parent_path.startswith(e.rstrip("/") + "/") for e in exclude)
+
+ ignore = base_ignore + [
e[len(path) :].lstrip("/")
for e in exclude
if mozpath.commonprefix((path, e)) == path
]
- finder = FileFinder(path, ignore=ignore, find_dotfiles=find_dotfiles)
-
- _, ext = os.path.splitext(path)
- ext.lstrip(".")
- for ext in extensions:
- for p, f in finder.find("**/*.{}".format(ext)):
+ finder = FileFinder(path, ignore=ignore, find_dotfiles=find_dotfiles)
+ if extensions:
+ for ext in extensions:
+ for p, f in finder.find("**/*.{}".format(ext)):
+ yield os.path.join(path, p)
+ else:
+ for p, f in finder.find("**/*.*"):
yield os.path.join(path, p)
diff --git a/python/mozlint/mozlint/types.py b/python/mozlint/mozlint/types.py
index 1a9a0bd473..468e28d81a 100644
--- a/python/mozlint/mozlint/types.py
+++ b/python/mozlint/mozlint/types.py
@@ -39,6 +39,7 @@ class BaseType(object):
config["include"],
config.get("exclude", []),
config.get("extensions", []),
+ config.get("exclude_extensions", []),
)
config["exclude"] = exclude
elif config.get("exclude"):
diff --git a/python/mozlint/test/linters/invalid_ext_and_exclude_ext.yml b/python/mozlint/test/linters/invalid_ext_and_exclude_ext.yml
new file mode 100644
index 0000000000..a73353fb71
--- /dev/null
+++ b/python/mozlint/test/linters/invalid_ext_and_exclude_ext.yml
@@ -0,0 +1,7 @@
+---
+InvalidExtensionAndExcludedExtensions:
+ type: string
+ payload: foobar
+ description: Having both extensions and exclude_extensions is not allowed.
+ extensions: ["*.js"]
+ exclude_extensions: ["*.png"]
diff --git a/python/mozlint/test/test_parser.py b/python/mozlint/test/test_parser.py
index 2fbf26c8e5..3cb319a2e0 100644
--- a/python/mozlint/test/test_parser.py
+++ b/python/mozlint/test/test_parser.py
@@ -59,6 +59,7 @@ def test_parser_valid_multiple(parse):
"invalid_include_with_glob.yml",
"invalid_exclude.yml",
"invalid_support_files.yml",
+ "invalid_ext_and_exclude_ext.yml",
"missing_attrs.yml",
"missing_definition.yml",
"non_existing_include.yml",
diff --git a/python/mozlint/test/test_pathutils.py b/python/mozlint/test/test_pathutils.py
index 78f7883e88..5f593fdc47 100644
--- a/python/mozlint/test/test_pathutils.py
+++ b/python/mozlint/test/test_pathutils.py
@@ -59,6 +59,43 @@ def assert_paths(a, b):
"extensions": ["py"],
"expected": ["a.py", "subdir1/b.py"],
},
+ pytest.param(
+ {
+ "paths": [
+ "a.py",
+ "a.js",
+ "subdir1/b.py",
+ "subdir2/c.py",
+ "subdir1/subdir3/d.py",
+ ],
+ "include": ["."],
+ "exclude": [],
+ "exclude_extensions": ["py"],
+ "expected": ["a.js"],
+ },
+ id="Excluding .py should only return .js file.",
+ ),
+ pytest.param(
+ {
+ "paths": [
+ "a.py",
+ "a.js",
+ "subdir1/b.py",
+ "subdir2/c.py",
+ "subdir1/subdir3/d.py",
+ ],
+ "include": ["."],
+ "exclude": [],
+ "exclude_extensions": ["js"],
+ "expected": [
+ "a.py",
+ "subdir1/b.py",
+ "subdir2/c.py",
+ "subdir1/subdir3/d.py",
+ ],
+ },
+ id="Excluding .js should only return .py files.",
+ ),
{
"paths": ["a.py", "a.js", "subdir2"],
"include": ["."],
@@ -104,24 +141,47 @@ def test_filterpaths(test):
"paths": ["subdir1/b.js"],
"config": {
"exclude": ["subdir1"],
- "extensions": "js",
+ "extensions": ["js"],
},
"expected": [],
},
{
- "paths": ["subdir1/subdir3"],
+ "paths": ["subdir1"],
"config": {
"exclude": ["subdir1"],
- "extensions": "js",
+ "extensions": ["js"],
},
"expected": [],
},
+ pytest.param(
+ {
+ "paths": ["a.py", "subdir1"],
+ "config": {
+ "exclude": ["subdir1"],
+ "exclude_extensions": ["gob"],
+ },
+ "expected": ["a.py"],
+ },
+ id="Excluding both subdirs and nonsense extensions returns other files.",
+ ),
+ pytest.param(
+ {
+ "paths": ["a.py", "a.js", "subdir1"],
+ "config": {
+ "exclude": [],
+ "exclude_extensions": ["py"],
+ },
+ "expected": ["a.js", "subdir1/subdir3/d.js", "subdir1/b.js"],
+ },
+ id="Excluding .py files returns only non-.py files, also from subdirs.",
+ ),
),
)
def test_expand_exclusions(test):
expected = test.pop("expected", [])
- paths = list(pathutils.expand_exclusions(test["paths"], test["config"], root))
+ input_paths = [os.path.join(root, p) for p in test["paths"]]
+ paths = list(pathutils.expand_exclusions(input_paths, test["config"], root))
assert_paths(paths, expected)
diff --git a/python/mozrelease/mozrelease/partner_repack.py b/python/mozrelease/mozrelease/partner_repack.py
index 1d64f43cca..77b42a83c9 100644
--- a/python/mozrelease/mozrelease/partner_repack.py
+++ b/python/mozrelease/mozrelease/partner_repack.py
@@ -172,7 +172,7 @@ def parseRepackConfig(file: Path, platform: str):
config[key] = value
continue
if key == "deb_section":
- config["deb_section"] = re.sub("/", "\/", value)
+ config["deb_section"] = re.sub("/", r"\/", value)
continue
if isValidPlatform(key):
ftp_platform = getFtpPlatform(key)
diff --git a/python/sites/build.txt b/python/sites/build.txt
index 07894139c0..18ed54c6a6 100644
--- a/python/sites/build.txt
+++ b/python/sites/build.txt
@@ -30,6 +30,7 @@ vendored:third_party/python/glean_parser
vendored:third_party/python/gyp/pylib
vendored:third_party/python/jinja2_time
vendored:third_party/python/json_e
+vendored:third_party/python/Mako
vendored:third_party/python/mohawk
vendored:third_party/python/mozilla_repo_urls
vendored:third_party/python/multidict
diff --git a/python/sites/docs.txt b/python/sites/docs.txt
index 42413388a6..9bfc36ba2c 100644
--- a/python/sites/docs.txt
+++ b/python/sites/docs.txt
@@ -4,7 +4,7 @@ pypi:boto3==1.33.5
pypi:fluent.pygments==1.0
pypi:livereload==2.6.3
pypi:mots==0.10.0
-pypi:myst-parser==1.0
+pypi:myst-parser==2.0
pypi:sphinx-copybutton==0.5.1
pypi:sphinx-design==0.5.0
pypi:sphinx-js==3.2.2
diff --git a/python/sites/mach.txt b/python/sites/mach.txt
index d9c98f24ce..e1cb8f159c 100644
--- a/python/sites/mach.txt
+++ b/python/sites/mach.txt
@@ -93,7 +93,7 @@ vendored:third_party/python/wheel
vendored:third_party/python/zipp
# glean-sdk may not be installable if a wheel isn't available
# and it has to be built from source.
-pypi-optional:glean-sdk==57.0.0:telemetry will not be collected
+pypi-optional:glean-sdk==58.1.0:telemetry will not be collected
# Mach gracefully handles the case where `psutil` is unavailable.
# We aren't (yet) able to pin packages in automation, so we have to
# support down to the oldest locally-installed version (5.4.2).