summaryrefslogtreecommitdiffstats
path: root/python/mozbuild
diff options
context:
space:
mode:
Diffstat (limited to 'python/mozbuild')
-rw-r--r--python/mozbuild/metrics.yaml18
-rw-r--r--python/mozbuild/mozbuild/action/buildlist.py3
-rw-r--r--python/mozbuild/mozbuild/action/dump_env.py16
-rw-r--r--python/mozbuild/mozbuild/action/l10n_merge.py2
-rw-r--r--python/mozbuild/mozbuild/action/test_archive.py2
-rwxr-xr-xpython/mozbuild/mozbuild/action/tooltool.py58
-rw-r--r--python/mozbuild/mozbuild/artifact_cache.py2
-rw-r--r--python/mozbuild/mozbuild/artifact_commands.py2
-rw-r--r--python/mozbuild/mozbuild/artifacts.py59
-rw-r--r--python/mozbuild/mozbuild/backend/common.py14
-rw-r--r--python/mozbuild/mozbuild/backend/fastermake.py10
-rw-r--r--python/mozbuild/mozbuild/backend/recursivemake.py27
-rw-r--r--python/mozbuild/mozbuild/backend/visualstudio.py3
-rw-r--r--python/mozbuild/mozbuild/buildversion.py23
-rw-r--r--python/mozbuild/mozbuild/config_status.py63
-rw-r--r--python/mozbuild/mozbuild/configure/lint.py25
-rw-r--r--python/mozbuild/mozbuild/controller/building.py25
-rw-r--r--python/mozbuild/mozbuild/dirutils.py52
-rw-r--r--python/mozbuild/mozbuild/doctor.py34
-rw-r--r--python/mozbuild/mozbuild/dotproperties.py8
-rw-r--r--python/mozbuild/mozbuild/frontend/data.py32
-rw-r--r--python/mozbuild/mozbuild/frontend/emitter.py4
-rw-r--r--python/mozbuild/mozbuild/gn_processor.py2
-rw-r--r--python/mozbuild/mozbuild/jar.py19
-rw-r--r--python/mozbuild/mozbuild/lock.py99
-rw-r--r--python/mozbuild/mozbuild/mach_commands.py19
-rw-r--r--python/mozbuild/mozbuild/makeutil.py23
-rw-r--r--python/mozbuild/mozbuild/mozconfig.py20
-rw-r--r--python/mozbuild/mozbuild/repackaging/deb.py40
-rw-r--r--python/mozbuild/mozbuild/repackaging/installer.py2
-rw-r--r--python/mozbuild/mozbuild/repackaging/mar.py2
-rw-r--r--python/mozbuild/mozbuild/repackaging/msi.py2
-rw-r--r--python/mozbuild/mozbuild/repackaging/msix.py2
-rw-r--r--python/mozbuild/mozbuild/test/backend/common.py9
-rw-r--r--python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/dist-bin/moz.build4
-rw-r--r--python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/dist-subdir/moz.build5
-rw-r--r--python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/final-target/moz.build5
-rw-r--r--python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/moz.build16
-rw-r--r--python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/not-installed/moz.build5
-rw-r--r--python/mozbuild/mozbuild/test/backend/test_build.py2
-rw-r--r--python/mozbuild/mozbuild/test/backend/test_recursivemake.py30
-rw-r--r--python/mozbuild/mozbuild/test/configure/test_checks_configure.py2
-rw-r--r--python/mozbuild/mozbuild/test/configure/test_configure.py44
-rw-r--r--python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/dist-bin/moz.build4
-rw-r--r--python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/dist-subdir/moz.build5
-rw-r--r--python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/final-target/moz.build5
-rw-r--r--python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/moz.build16
-rw-r--r--python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/not-installed/moz.build5
-rw-r--r--python/mozbuild/mozbuild/test/frontend/test_emitter.py16
-rw-r--r--python/mozbuild/mozbuild/test/test_containers.py7
-rw-r--r--python/mozbuild/mozbuild/toolchains.py4
-rw-r--r--python/mozbuild/mozbuild/util.py169
-rw-r--r--python/mozbuild/mozbuild/vendor/rewrite_mozbuild.py21
-rw-r--r--python/mozbuild/mozbuild/vendor/vendor_python.py5
-rw-r--r--python/mozbuild/mozpack/dmg.py2
-rw-r--r--python/mozbuild/mozpack/test/test_files.py3
-rw-r--r--python/mozbuild/mozpack/test/test_unify.py2
57 files changed, 637 insertions, 461 deletions
diff --git a/python/mozbuild/metrics.yaml b/python/mozbuild/metrics.yaml
index 068dd6a389..5c81240302 100644
--- a/python/mozbuild/metrics.yaml
+++ b/python/mozbuild/metrics.yaml
@@ -22,7 +22,7 @@ mozbuild:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
notification_emails:
- build-telemetry@mozilla.com
- - mhentges@mozilla.com
+ - ahochheiden@mozilla.com
expires: never
send_in_pings:
- usage
@@ -36,7 +36,7 @@ mozbuild:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
notification_emails:
- build-telemetry@mozilla.com
- - mhentges@mozilla.com
+ - ahochheiden@mozilla.com
expires: never
send_in_pings:
- usage
@@ -50,7 +50,7 @@ mozbuild:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
notification_emails:
- build-telemetry@mozilla.com
- - mhentges@mozilla.com
+ - ahochheiden@mozilla.com
expires: never
send_in_pings:
- usage
@@ -64,7 +64,7 @@ mozbuild:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
notification_emails:
- build-telemetry@mozilla.com
- - mhentges@mozilla.com
+ - ahochheiden@mozilla.com
expires: never
send_in_pings:
- usage
@@ -78,7 +78,7 @@ mozbuild:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
notification_emails:
- build-telemetry@mozilla.com
- - mhentges@mozilla.com
+ - ahochheiden@mozilla.com
expires: never
send_in_pings:
- usage
@@ -92,7 +92,7 @@ mozbuild:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
notification_emails:
- build-telemetry@mozilla.com
- - mhentges@mozilla.com
+ - ahochheiden@mozilla.com
expires: never
send_in_pings:
- usage
@@ -106,7 +106,7 @@ mozbuild:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
notification_emails:
- build-telemetry@mozilla.com
- - mhentges@mozilla.com
+ - ahochheiden@mozilla.com
expires: never
send_in_pings:
- usage
@@ -120,7 +120,7 @@ mozbuild:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1526072#c15
notification_emails:
- build-telemetry@mozilla.com
- - mhentges@mozilla.com
+ - ahochheiden@mozilla.com
expires: never
send_in_pings:
- usage
@@ -134,7 +134,7 @@ mozbuild:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1654084#c2
notification_emails:
- build-telemetry@mozilla.com
- - mhentges@mozilla.com
+ - ahochheiden@mozilla.com
expires: never
send_in_pings:
- usage
diff --git a/python/mozbuild/mozbuild/action/buildlist.py b/python/mozbuild/mozbuild/action/buildlist.py
index ed7149eb87..b0514053ce 100644
--- a/python/mozbuild/mozbuild/action/buildlist.py
+++ b/python/mozbuild/mozbuild/action/buildlist.py
@@ -11,7 +11,8 @@ import io
import os
import sys
-from mozbuild.util import ensureParentDir, lock_file
+from mozbuild.dirutils import ensureParentDir
+from mozbuild.lock import lock_file
def addEntriesToListFile(listFile, entries):
diff --git a/python/mozbuild/mozbuild/action/dump_env.py b/python/mozbuild/mozbuild/action/dump_env.py
index ec178700eb..de62089afd 100644
--- a/python/mozbuild/mozbuild/action/dump_env.py
+++ b/python/mozbuild/mozbuild/action/dump_env.py
@@ -12,19 +12,5 @@ sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
from shellutil import quote
-
-def environ():
- # We would use six.ensure_text but the global Python isn't guaranteed to have
- # the correct version of six installed.
- def ensure_text(s):
- if sys.version_info > (3, 0) or isinstance(s, unicode):
- # os.environ always returns string keys and values in Python 3.
- return s
- else:
- return s.decode("utf-8")
-
- return [(ensure_text(k), ensure_text(v)) for (k, v) in os.environ.items()]
-
-
-for key, value in environ():
+for key, value in os.environ.items():
print("%s=%s" % (key, quote(value)))
diff --git a/python/mozbuild/mozbuild/action/l10n_merge.py b/python/mozbuild/mozbuild/action/l10n_merge.py
index 1a04d60107..84975497fe 100644
--- a/python/mozbuild/mozbuild/action/l10n_merge.py
+++ b/python/mozbuild/mozbuild/action/l10n_merge.py
@@ -7,7 +7,7 @@ import os
import shutil
import sys
-from mozbuild.util import ensureParentDir
+from mozbuild.dirutils import ensureParentDir
def main(argv):
diff --git a/python/mozbuild/mozbuild/action/test_archive.py b/python/mozbuild/mozbuild/action/test_archive.py
index 1e2fcc15c9..f5dd893d3b 100644
--- a/python/mozbuild/mozbuild/action/test_archive.py
+++ b/python/mozbuild/mozbuild/action/test_archive.py
@@ -24,7 +24,7 @@ from mozpack.manifests import InstallManifest
from mozpack.mozjar import JarWriter
from reftest import ReftestManifest
-from mozbuild.util import ensureParentDir
+from mozbuild.dirutils import ensureParentDir
STAGE = mozpath.join(buildconfig.topobjdir, "dist", "test-stage")
diff --git a/python/mozbuild/mozbuild/action/tooltool.py b/python/mozbuild/mozbuild/action/tooltool.py
index 6b53db31e8..9abbff019e 100755
--- a/python/mozbuild/mozbuild/action/tooltool.py
+++ b/python/mozbuild/mozbuild/action/tooltool.py
@@ -61,25 +61,12 @@ REQUEST_HEADER_ATTRIBUTE_CHARS = re.compile(
DEFAULT_MANIFEST_NAME = "manifest.tt"
TOOLTOOL_PACKAGE_SUFFIX = ".TOOLTOOL-PACKAGE"
HAWK_VER = 1
-PY3 = sys.version_info[0] == 3
-
-if PY3:
- six_binary_type = bytes
- unicode = (
- str # Silence `pyflakes` from reporting `undefined name 'unicode'` in Python 3.
- )
- import urllib.request as urllib2
- from http.client import HTTPConnection, HTTPSConnection
- from urllib.error import HTTPError, URLError
- from urllib.parse import urljoin, urlparse
- from urllib.request import Request
-else:
- six_binary_type = str
- import urllib2
- from httplib import HTTPConnection, HTTPSConnection
- from urllib2 import HTTPError, Request, URLError
- from urlparse import urljoin, urlparse
+import urllib.request as urllib2
+from http.client import HTTPConnection, HTTPSConnection
+from urllib.error import HTTPError, URLError
+from urllib.parse import urljoin, urlparse
+from urllib.request import Request
log = logging.getLogger(__name__)
@@ -205,9 +192,7 @@ def retriable(*retry_args, **retry_kwargs):
def request_has_data(req):
- if PY3:
- return req.data is not None
- return req.has_data()
+ return req.data is not None
def get_hexdigest(val):
@@ -281,7 +266,7 @@ def random_string(length):
def prepare_header_val(val):
- if isinstance(val, six_binary_type):
+ if isinstance(val, bytes):
val = val.decode("utf-8")
if not REQUEST_HEADER_ATTRIBUTE_CHARS.match(val):
@@ -303,7 +288,7 @@ def parse_content_type(content_type): # pragma: no cover
def calculate_payload_hash(algorithm, payload, content_type): # pragma: no cover
parts = [
- part if isinstance(part, six_binary_type) else part.encode("utf8")
+ part if isinstance(part, bytes) else part.encode("utf8")
for part in [
"hawk." + str(HAWK_VER) + ".payload\n",
parse_content_type(content_type) + "\n",
@@ -337,7 +322,7 @@ def validate_taskcluster_credentials(credentials):
def normalize_header_attr(val):
- if isinstance(val, six_binary_type):
+ if isinstance(val, bytes):
return val.decode("utf-8")
return val # pragma: no cover
@@ -390,10 +375,10 @@ def calculate_mac(
log.debug("normalized resource for mac calc: {norm}".format(norm=normalized))
digestmod = getattr(hashlib, algorithm)
- if not isinstance(normalized, six_binary_type):
+ if not isinstance(normalized, bytes):
normalized = normalized.encode("utf8")
- if not isinstance(access_token, six_binary_type):
+ if not isinstance(access_token, bytes):
access_token = access_token.encode("ascii")
result = hmac.new(access_token, normalized, digestmod)
@@ -412,10 +397,7 @@ def make_taskcluster_header(credentials, req):
content_hash = None
if request_has_data(req):
- if PY3:
- data = req.data
- else:
- data = req.get_data()
+ data = req.data
content_hash = calculate_payload_hash( # pragma: no cover
algorithm,
data,
@@ -760,7 +742,7 @@ def open_manifest(manifest_file):
"""I know how to take a filename and load it into a Manifest object"""
if os.path.exists(manifest_file):
manifest = Manifest()
- with open(manifest_file, "r" if PY3 else "rb") as f:
+ with open(manifest_file, "r") as f:
manifest.load(f)
log.debug("loaded manifest from file '%s'" % manifest_file)
return manifest
@@ -865,12 +847,10 @@ def add_files(manifest_file, algorithm, filenames, version, visibility, unpack):
for old_fr in old_manifest.file_records:
if old_fr.filename not in new_filenames:
new_manifest.file_records.append(old_fr)
- if PY3:
- with open(manifest_file, mode="w") as output:
- new_manifest.dump(output, fmt="json")
- else:
- with open(manifest_file, mode="wb") as output:
- new_manifest.dump(output, fmt="json")
+
+ with open(manifest_file, mode="w") as output:
+ new_manifest.dump(output, fmt="json")
+
return all_files_added
@@ -1288,9 +1268,7 @@ def _send_batch(base_url, auth_file, batch, region):
url = urljoin(base_url, "upload")
if region is not None:
url += "?region=" + region
- data = json.dumps(batch)
- if PY3:
- data = data.encode("utf-8")
+ data = json.dumps(batch).encode("utf-8")
req = Request(url, data, {"Content-Type": "application/json"})
_authorize(req, auth_file)
try:
diff --git a/python/mozbuild/mozbuild/artifact_cache.py b/python/mozbuild/mozbuild/artifact_cache.py
index 572953e1f7..bd6dd89697 100644
--- a/python/mozbuild/mozbuild/artifact_cache.py
+++ b/python/mozbuild/mozbuild/artifact_cache.py
@@ -31,7 +31,7 @@ import mozpack.path as mozpath
import six
import six.moves.urllib.parse as urlparse
-from mozbuild.util import mkdir
+from mozbuild.dirutils import mkdir
# Using 'DownloadManager' through the provided interface we
# can't directly specify a 'chunk_size' for the 'Download' it manages.
diff --git a/python/mozbuild/mozbuild/artifact_commands.py b/python/mozbuild/mozbuild/artifact_commands.py
index 62406f406a..e42b5a38d9 100644
--- a/python/mozbuild/mozbuild/artifact_commands.py
+++ b/python/mozbuild/mozbuild/artifact_commands.py
@@ -22,7 +22,7 @@ from mach.decorators import Command, CommandArgument, SubCommand
from mozbuild.artifact_builds import JOB_CHOICES
from mozbuild.base import MachCommandConditions as conditions
-from mozbuild.util import ensureParentDir
+from mozbuild.dirutils import ensureParentDir
_COULD_NOT_FIND_ARTIFACTS_TEMPLATE = (
"ERROR!!!!!! Could not find artifacts for a toolchain build named "
diff --git a/python/mozbuild/mozbuild/artifacts.py b/python/mozbuild/mozbuild/artifacts.py
index eff16f6c9b..26ef99b1a2 100644
--- a/python/mozbuild/mozbuild/artifacts.py
+++ b/python/mozbuild/mozbuild/artifacts.py
@@ -65,7 +65,8 @@ from taskgraph.util.taskcluster import find_task_id, get_artifact_url, list_arti
from mozbuild.artifact_builds import JOB_CHOICES
from mozbuild.artifact_cache import ArtifactCache
-from mozbuild.util import FileAvoidWrite, ensureParentDir, mkdir
+from mozbuild.dirutils import ensureParentDir, mkdir
+from mozbuild.util import FileAvoidWrite
# Number of candidate pushheads to cache per parent changeset.
NUM_PUSHHEADS_TO_QUERY_PER_PARENT = 50
@@ -151,14 +152,19 @@ class ArtifactJob(object):
# prepended.
#
# The entries in the archive, suitably renamed, will be extracted into `dist`.
- _extra_archives = {
- ".xpt_artifacts.zip": {
- "description": "XPT Artifacts",
- "src_prefix": "",
- "dest_prefix": "xpt_artifacts",
- },
- }
- _extra_archive_suffixes = tuple(sorted(_extra_archives.keys()))
+ @property
+ def _extra_archives(self):
+ return {
+ ".xpt_artifacts.zip": {
+ "description": "XPT Artifacts",
+ "src_prefix": "",
+ "dest_prefix": "xpt_artifacts",
+ },
+ }
+
+ @property
+ def _extra_archive_suffixes(self):
+ return tuple(sorted(self._extra_archives.keys()))
def __init__(
self,
@@ -213,7 +219,7 @@ class ArtifactJob(object):
self._symbols_archive_suffix
):
yield name
- elif name.endswith(ArtifactJob._extra_archive_suffixes):
+ elif name.endswith(self._extra_archive_suffixes):
yield name
else:
self.log(
@@ -247,7 +253,7 @@ class ArtifactJob(object):
self._symbols_archive_suffix
):
return self.process_symbols_archive(filename, processed_filename)
- if filename.endswith(ArtifactJob._extra_archive_suffixes):
+ if filename.endswith(self._extra_archive_suffixes):
return self.process_extra_archive(filename, processed_filename)
return self.process_package_artifact(filename, processed_filename)
@@ -401,7 +407,7 @@ class ArtifactJob(object):
writer.add(destpath.encode("utf-8"), entry)
def process_extra_archive(self, filename, processed_filename):
- for suffix, extra_archive in ArtifactJob._extra_archives.items():
+ for suffix, extra_archive in self._extra_archives.items():
if filename.endswith(suffix):
self.log(
logging.INFO,
@@ -657,21 +663,28 @@ class MacArtifactJob(ArtifactJob):
"nmhproxy",
"pingsender",
"plugin-container.app/Contents/MacOS/plugin-container",
- "updater.app/Contents/Frameworks/UpdateSettings.framework/UpdateSettings",
"updater.app/Contents/MacOS/org.mozilla.updater",
# 'xpcshell',
"XUL",
],
),
- (
- "Contents/Frameworks",
- [
- "ChannelPrefs.framework/ChannelPrefs",
- ],
- ),
)
@property
+ def _extra_archives(self):
+ extra_archives = super()._extra_archives
+ extra_archives.update(
+ {
+ ".update_framework_artifacts.zip": {
+ "description": "Update-related macOS Framework Artifacts",
+ "src_prefix": "",
+ "dest_prefix": "update_framework_artifacts",
+ },
+ }
+ )
+ return extra_archives
+
+ @property
def paths_no_keep_path(self):
formatted = []
for root, paths in self._paths_no_keep_path:
@@ -1275,8 +1288,8 @@ class Artifacts(object):
zeroes = "0" * 40
hashes = []
- for hg_hash in hg_hash_list.splitlines():
- hg_hash = hg_hash.strip()
+ for hg_hash_unstripped in hg_hash_list.splitlines():
+ hg_hash = hg_hash_unstripped.strip()
if not hg_hash or hg_hash == zeroes:
continue
hashes.append(hg_hash)
@@ -1388,8 +1401,8 @@ https://firefox-source-docs.mozilla.org/contributing/vcs/mercurial_bundles.html
if candidate_pushheads:
break
count = 0
- for rev in last_revs:
- rev = rev.rstrip()
+ for rev_unstripped in last_revs:
+ rev = rev_unstripped.rstrip()
if not rev:
continue
if rev not in candidate_pushheads:
diff --git a/python/mozbuild/mozbuild/backend/common.py b/python/mozbuild/mozbuild/backend/common.py
index b1c90f31fd..3e14484a30 100644
--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -13,6 +13,7 @@ import six
from mozpack.chrome.manifest import parse_manifest_line
from mozbuild.backend.base import BuildBackend
+from mozbuild.dirutils import mkdir
from mozbuild.frontend.context import (
VARIABLES,
Context,
@@ -44,7 +45,6 @@ from mozbuild.frontend.data import (
)
from mozbuild.jar import DeprecatedJarManifest, JarManifestParser
from mozbuild.preprocessor import Preprocessor
-from mozbuild.util import mkdir
class XPIDLManager(object):
@@ -182,7 +182,7 @@ class CommonBackend(BuildBackend):
# the order is not consistent across multiple runs.
#
# Exclude this file in order to avoid breaking the
- # taskcluster/ci/diffoscope/reproducible.yml jobs.
+ # taskcluster/kinds/diffoscope/reproducible.yml jobs.
continue
fullpath = ObjDirPath(obj._context, "!" + f).full_path
self._handle_generated_sources([fullpath])
@@ -325,15 +325,13 @@ class CommonBackend(BuildBackend):
list_style = "list"
list_file_path = mozpath.join(objdir, name)
objs = [os.path.relpath(o, objdir) for o in objs]
- if list_style == "linkerscript":
- ref = list_file_path
- content = "\n".join('INPUT("%s")' % o for o in objs)
- elif list_style == "filelist":
+ content = "\n".join(objs)
+ if list_style == "filelist":
ref = "-Wl,-filelist," + list_file_path
- content = "\n".join(objs)
+ elif list_style == "linkerlist":
+ ref = "-Wl,@" + list_file_path
elif list_style == "list":
ref = "@" + list_file_path
- content = "\n".join(objs)
else:
return None
diff --git a/python/mozbuild/mozbuild/backend/fastermake.py b/python/mozbuild/mozbuild/backend/fastermake.py
index c423e00c32..3b0927defa 100644
--- a/python/mozbuild/mozbuild/backend/fastermake.py
+++ b/python/mozbuild/mozbuild/backend/fastermake.py
@@ -2,6 +2,7 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from collections import defaultdict
from operator import itemgetter
import mozpack.path as mozpath
@@ -22,19 +23,18 @@ from mozbuild.frontend.data import (
XPIDLModule,
)
from mozbuild.makeutil import Makefile
-from mozbuild.util import OrderedDefaultDict
class FasterMakeBackend(MakeBackend, PartialBackend):
def _init(self):
super(FasterMakeBackend, self)._init()
- self._manifest_entries = OrderedDefaultDict(set)
+ self._manifest_entries = defaultdict(set)
- self._install_manifests = OrderedDefaultDict(InstallManifest)
+ self._install_manifests = defaultdict(InstallManifest)
- self._dependencies = OrderedDefaultDict(list)
- self._l10n_dependencies = OrderedDefaultDict(list)
+ self._dependencies = defaultdict(list)
+ self._l10n_dependencies = defaultdict(list)
self._has_xpidl = False
diff --git a/python/mozbuild/mozbuild/backend/recursivemake.py b/python/mozbuild/mozbuild/backend/recursivemake.py
index 86cb6ccc10..e3b5649d7e 100644
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -69,7 +69,7 @@ from ..frontend.data import (
XPIDLModule,
)
from ..makeutil import Makefile
-from ..util import FileAvoidWrite, OrderedDefaultDict, ensureParentDir, pairwise
+from ..util import FileAvoidWrite, ensureParentDir, pairwise
from .common import CommonBackend
from .make import MakeBackend
@@ -364,7 +364,7 @@ class RecursiveMakeBackend(MakeBackend):
self._install_manifests["dist_private"]
self._traversal = RecursiveMakeTraversal()
- self._compile_graph = OrderedDefaultDict(set)
+ self._compile_graph = defaultdict(set)
self._rust_targets = set()
self._gkrust_target = None
self._pre_compile = set()
@@ -810,7 +810,7 @@ class RecursiveMakeBackend(MakeBackend):
rule.add_dependencies(sorted(deps))
non_default_roots = defaultdict(list)
- non_default_graphs = defaultdict(lambda: OrderedDefaultDict(set))
+ non_default_graphs = defaultdict(lambda: defaultdict(set))
for root in compile_roots:
# If this is a non-default target, separate the root from the
@@ -1366,7 +1366,8 @@ class RecursiveMakeBackend(MakeBackend):
backend_file.write_once("LIBRARY_NAME := %s\n" % libdef.basename)
backend_file.write("FORCE_SHARED_LIB := 1\n")
backend_file.write("IMPORT_LIBRARY := %s\n" % libdef.import_name)
- backend_file.write("SHARED_LIBRARY := %s\n" % libdef.lib_name)
+ shared_lib = self._pretty_path(libdef.output_path, backend_file)
+ backend_file.write("SHARED_LIBRARY := %s\n" % shared_lib)
if libdef.soname:
backend_file.write("DSO_SONAME := %s\n" % libdef.soname)
if libdef.symbols_file:
@@ -1375,11 +1376,7 @@ class RecursiveMakeBackend(MakeBackend):
if not libdef.cxx_link:
backend_file.write("LIB_IS_C_ONLY := 1\n")
if libdef.output_category:
- self._process_non_default_target(libdef, libdef.lib_name, backend_file)
- # Override the install rule target for this library. This is hacky,
- # but can go away as soon as we start building libraries in their
- # final location (bug 1459764).
- backend_file.write("SHARED_LIBRARY_TARGET := %s\n" % libdef.output_category)
+ self._process_non_default_target(libdef, shared_lib, backend_file)
def _process_static_library(self, libdef, backend_file):
backend_file.write_once("LIBRARY_NAME := %s\n" % libdef.basename)
@@ -1427,15 +1424,13 @@ class RecursiveMakeBackend(MakeBackend):
)
def _process_linked_libraries(self, obj, backend_file):
- def pretty_relpath(lib, name):
- return os.path.normpath(
- mozpath.join(mozpath.relpath(lib.objdir, obj.objdir), name)
- )
+ def pretty_relpath(path):
+ return os.path.normpath(mozpath.relpath(path, obj.objdir))
objs, shared_libs, os_libs, static_libs = self._expand_libs(obj)
obj_target = obj.name
- if isinstance(obj, Program):
+ if isinstance(obj, (Program, SharedLibrary)):
obj_target = self._pretty_path(obj.output_path, backend_file)
objs_ref = " \\\n ".join(os.path.relpath(o, obj.objdir) for o in objs)
@@ -1485,7 +1480,7 @@ class RecursiveMakeBackend(MakeBackend):
for lib in shared_libs:
assert obj.KIND != "host" and obj.KIND != "wasm"
backend_file.write_once(
- "SHARED_LIBS += %s\n" % pretty_relpath(lib, lib.import_name)
+ "SHARED_LIBS += %s\n" % pretty_relpath(lib.import_path)
)
# We have to link any Rust libraries after all intermediate static
@@ -1497,7 +1492,7 @@ class RecursiveMakeBackend(MakeBackend):
(l for l in static_libs if isinstance(l, BaseRustLibrary)),
):
backend_file.write_once(
- "%s += %s\n" % (var, pretty_relpath(lib, lib.import_name))
+ "%s += %s\n" % (var, pretty_relpath(lib.import_path))
)
for lib in os_libs:
diff --git a/python/mozbuild/mozbuild/backend/visualstudio.py b/python/mozbuild/mozbuild/backend/visualstudio.py
index ccf3c65a68..1b82bafbc8 100644
--- a/python/mozbuild/mozbuild/backend/visualstudio.py
+++ b/python/mozbuild/mozbuild/backend/visualstudio.py
@@ -8,7 +8,6 @@
import errno
import os
import re
-import sys
import uuid
from pathlib import Path
from xml.dom import getDOMImplementation
@@ -35,8 +34,6 @@ MSNATVIS_NAMESPACE = "http://schemas.microsoft.com/vstudio/debugger/natvis/2010"
def get_id(name):
- if sys.version_info[0] == 2:
- name = name.encode("utf-8")
return str(uuid.uuid5(uuid.NAMESPACE_URL, name)).upper()
diff --git a/python/mozbuild/mozbuild/buildversion.py b/python/mozbuild/mozbuild/buildversion.py
new file mode 100644
index 0000000000..a0a767ab82
--- /dev/null
+++ b/python/mozbuild/mozbuild/buildversion.py
@@ -0,0 +1,23 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+from pathlib import Path
+
+from packaging.version import Version
+
+
+def mozilla_build_version():
+ mozilla_build = os.environ.get("MOZILLABUILD")
+
+ version_file = Path(mozilla_build) / "VERSION"
+
+ assert version_file.exists(), (
+ f'The MozillaBuild VERSION file was not found at "{version_file}".\n'
+ "Please check if MozillaBuild is installed correctly and that the"
+ "`MOZILLABUILD` environment variable is to the correct path."
+ )
+
+ with version_file.open() as file:
+ return Version(file.readline().rstrip("\n"))
diff --git a/python/mozbuild/mozbuild/config_status.py b/python/mozbuild/mozbuild/config_status.py
index 8e8a7f625b..f99d0c5d3b 100644
--- a/python/mozbuild/mozbuild/config_status.py
+++ b/python/mozbuild/mozbuild/config_status.py
@@ -12,6 +12,8 @@ import sys
import time
from argparse import ArgumentParser
from itertools import chain
+from multiprocessing import Pool, get_start_method
+from time import process_time
from mach.logging import LoggingManager
@@ -21,7 +23,7 @@ from mozbuild.base import MachCommandConditions
from mozbuild.frontend.emitter import TreeMetadataEmitter
from mozbuild.frontend.reader import BuildReader
from mozbuild.mozinfo import write_mozinfo
-from mozbuild.util import FileAvoidWrite, process_time
+from mozbuild.util import FileAvoidWrite
log_manager = LoggingManager()
@@ -39,6 +41,45 @@ https://firefox-source-docs.mozilla.org/mobile/android/geckoview/contributor/gec
""".strip()
+## Parallel backend setup
+# Distributing each backend on different process is costly because we need to
+# copy the definitions across each process. These definitions are read-only, so
+# only copy them once when each process starts.
+
+
+class BackendPool:
+ per_process_definitions = None
+
+ def __init__(self, definitions, *, processes=None):
+ definitions = list(definitions)
+ BackendPool._init_worker(definitions)
+ self.pool = Pool(
+ initializer=BackendPool._init_worker,
+ initargs=(definitions,),
+ processes=processes,
+ )
+
+ def run(self, backends):
+ # We're trying to spawn a minimal number of new processes there, and
+ # limit the number of times we serialize the task state. As a
+ # consequence:
+ # 1. we initialize each process with a copy of `definitions'
+ # 2. instead of spawning as many processes as backend, we use current
+ # process to handle one of the backend and asynchronously run the
+ # others.
+ async_tasks = self.pool.map_async(BackendPool._run_worker, backends[1:])
+ BackendPool._run_worker(backends[0])
+ async_tasks.wait()
+
+ @staticmethod
+ def _init_worker(state):
+ BackendPool.per_process_definitions = state
+
+ @staticmethod
+ def _run_worker(backend):
+ return backend.consume(BackendPool.per_process_definitions)
+
+
def config_status(
topobjdir=".",
topsrcdir=".",
@@ -148,11 +189,23 @@ def config_status(
log_manager.enable_unstructured()
print("Reticulating splines...", file=sys.stderr)
- if len(selected_backends) > 1:
- definitions = list(definitions)
- for the_backend in selected_backends:
- the_backend.consume(definitions)
+ # `definitions` objects are unfortunately not picklable, which is a
+ # requirement for "spawn" method. It's fine under "fork" method. This
+ # basically excludes Windows from our optimization, we can live with it.
+ if len(selected_backends) > 1 and get_start_method() == "fork":
+ # See https://github.com/python/cpython/commit/39889864c09741909da4ec489459d0197ea8f1fc
+ # For why we cap the process count. There's also an overhead to setup
+ # new processes, and not that many backends anyway.
+ processes = min(len(selected_backends) - 1, 4)
+ pool = BackendPool(definitions, processes=processes)
+ pool.run(selected_backends)
+ else:
+ if len(selected_backends) > 1:
+ definitions = list(definitions)
+
+ for backend in selected_backends:
+ backend.consume(definitions)
execution_time = 0.0
for obj in chain((reader, emitter), selected_backends):
diff --git a/python/mozbuild/mozbuild/configure/lint.py b/python/mozbuild/mozbuild/configure/lint.py
index c6bda9a540..4858188964 100644
--- a/python/mozbuild/mozbuild/configure/lint.py
+++ b/python/mozbuild/mozbuild/configure/lint.py
@@ -4,7 +4,6 @@
import inspect
import re
-import sys
import types
from dis import Bytecode
from functools import wraps
@@ -25,27 +24,9 @@ from .help import HelpFormatter
def code_replace(code, co_filename, co_name, co_firstlineno):
- if sys.version_info < (3, 8):
- codetype_args = [
- code.co_argcount,
- code.co_kwonlyargcount,
- code.co_nlocals,
- code.co_stacksize,
- code.co_flags,
- code.co_code,
- code.co_consts,
- code.co_names,
- code.co_varnames,
- co_filename,
- co_name,
- co_firstlineno,
- code.co_lnotab,
- ]
- return types.CodeType(*codetype_args)
- else:
- return code.replace(
- co_filename=co_filename, co_name=co_name, co_firstlineno=co_firstlineno
- )
+ return code.replace(
+ co_filename=co_filename, co_name=co_name, co_firstlineno=co_firstlineno
+ )
class LintSandbox(ConfigureSandbox):
diff --git a/python/mozbuild/mozbuild/controller/building.py b/python/mozbuild/mozbuild/controller/building.py
index 30f8136f26..75f8099672 100644
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -34,9 +34,10 @@ from mozterm.widgets import Footer
from ..backend import get_backend_class
from ..base import MozbuildObject
from ..compilation.warnings import WarningsCollector, WarningsDatabase
+from ..dirutils import mkdir
from ..telemetry import get_cpu_brand
from ..testing import install_test_files
-from ..util import FileAvoidWrite, mkdir, resolve_target_to_make
+from ..util import FileAvoidWrite, resolve_target_to_make
from .clobber import Clobberer
FINDER_SLOW_MESSAGE = """
@@ -262,6 +263,12 @@ class BuildMonitor(MozbuildObject):
_, _, disambiguator = args.pop(0).partition("@")
action = args.pop(0)
+ time = None
+ regexp = re.compile(r"\d{10}(\.\d{1,9})?$")
+ if regexp.match(action):
+ time = float(action)
+ action = args.pop(0)
+
update_needed = True
if action == "TIERS":
@@ -279,12 +286,12 @@ class BuildMonitor(MozbuildObject):
update_needed = False
elif action.startswith("START_"):
self.resources.begin_marker(
- action[len("START_") :], " ".join(args), disambiguator
+ action[len("START_") :], " ".join(args), disambiguator, time
)
update_needed = False
elif action.startswith("END_"):
self.resources.end_marker(
- action[len("END_") :], " ".join(args), disambiguator
+ action[len("END_") :], " ".join(args), disambiguator, time
)
update_needed = False
elif action == "BUILD_VERBOSE":
@@ -678,14 +685,14 @@ class BuildOutputManager(OutputManager):
if message:
self.log(logging.INFO, "build_output", {"line": message}, "{line}")
elif state_changed:
- have_handler = hasattr(self, "handler")
+ have_handler = hasattr(self, "_handler")
if have_handler:
- self.handler.acquire()
+ self._handler.acquire()
try:
self.refresh()
finally:
if have_handler:
- self.handler.release()
+ self._handler.release()
class StaticAnalysisFooter(Footer):
@@ -740,14 +747,14 @@ class StaticAnalysisOutputManager(OutputManager):
if relevant:
self.log(logging.INFO, "build_output", {"line": line}, "{line}")
else:
- have_handler = hasattr(self, "handler")
+ have_handler = hasattr(self, "_handler")
if have_handler:
- self.handler.acquire()
+ self._handler.acquire()
try:
self.refresh()
finally:
if have_handler:
- self.handler.release()
+ self._handler.release()
def write(self, path, output_format):
assert output_format in ("text", "json"), "Invalid output format {}".format(
diff --git a/python/mozbuild/mozbuild/dirutils.py b/python/mozbuild/mozbuild/dirutils.py
new file mode 100644
index 0000000000..efa4746301
--- /dev/null
+++ b/python/mozbuild/mozbuild/dirutils.py
@@ -0,0 +1,52 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This file contains miscellaneous utility functions that don't belong anywhere
+# in particular.
+
+import errno
+import os
+import sys
+
+if sys.platform == "win32":
+ import ctypes
+
+ _kernel32 = ctypes.windll.kernel32
+ _FILE_ATTRIBUTE_NOT_CONTENT_INDEXED = 0x2000
+
+
+def ensureParentDir(path):
+ """Ensures the directory parent to the given file exists."""
+ d = os.path.dirname(path)
+ if d and not os.path.exists(path):
+ try:
+ os.makedirs(d)
+ except OSError as error:
+ if error.errno != errno.EEXIST:
+ raise
+
+
+def mkdir(path, not_indexed=False):
+ """Ensure a directory exists.
+
+ If ``not_indexed`` is True, an attribute is set that disables content
+ indexing on the directory.
+ """
+ try:
+ os.makedirs(path)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+
+ if not_indexed:
+ if sys.platform == "win32":
+ if isinstance(path, str):
+ fn = _kernel32.SetFileAttributesW
+ else:
+ fn = _kernel32.SetFileAttributesA
+
+ fn(path, _FILE_ATTRIBUTE_NOT_CONTENT_INDEXED)
+ elif sys.platform == "darwin":
+ with open(os.path.join(path, ".metadata_never_index"), "a"):
+ pass
diff --git a/python/mozbuild/mozbuild/doctor.py b/python/mozbuild/mozbuild/doctor.py
index 315a00e7c0..2bcb91042e 100644
--- a/python/mozbuild/mozbuild/doctor.py
+++ b/python/mozbuild/mozbuild/doctor.py
@@ -3,7 +3,6 @@
# file, # You can obtain one at http://mozilla.org/MPL/2.0/.
import enum
-import locale
import os
import socket
import subprocess
@@ -460,39 +459,6 @@ def mozillabuild(**kwargs) -> DoctorCheck:
@check
-def bad_locale_utf8(**kwargs) -> DoctorCheck:
- """Check to detect the invalid locale `UTF-8` on pre-3.8 Python."""
- if sys.version_info >= (3, 8):
- return DoctorCheck(
- name="utf8 locale",
- status=CheckStatus.SKIPPED,
- display_text=["Python version has fixed utf-8 locale bug."],
- )
-
- try:
- # This line will attempt to get and parse the locale.
- locale.getdefaultlocale()
-
- return DoctorCheck(
- name="utf8 locale",
- status=CheckStatus.OK,
- display_text=["Python's locale is set to a valid value."],
- )
- except ValueError:
- return DoctorCheck(
- name="utf8 locale",
- status=CheckStatus.FATAL,
- display_text=[
- "Your Python is using an invalid value for its locale.",
- "Either update Python to version 3.8+, or set the following variables in ",
- "your environment:",
- " export LC_ALL=en_US.UTF-8",
- " export LANG=en_US.UTF-8",
- ],
- )
-
-
-@check
def artifact_build(
topsrcdir: str, configure_args: Optional[List[str]], **kwargs
) -> DoctorCheck:
diff --git a/python/mozbuild/mozbuild/dotproperties.py b/python/mozbuild/mozbuild/dotproperties.py
index c02e1a794b..e65ecd8301 100644
--- a/python/mozbuild/mozbuild/dotproperties.py
+++ b/python/mozbuild/mozbuild/dotproperties.py
@@ -6,15 +6,9 @@
import codecs
import re
-import sys
import six
-if sys.version_info[0] == 3:
- str_type = str
-else:
- str_type = basestring
-
class DotProperties:
r"""A thin representation of a key=value .properties file."""
@@ -29,7 +23,7 @@ class DotProperties:
Ignores empty lines and comment lines."""
- if isinstance(file, str_type):
+ if isinstance(file, str):
f = codecs.open(file, "r", "utf-8")
else:
f = file
diff --git a/python/mozbuild/mozbuild/frontend/data.py b/python/mozbuild/mozbuild/frontend/data.py
index 4217196400..b958680aa5 100644
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -391,6 +391,15 @@ class Linkable(ContextDerived):
self.lib_defines = Defines(context, OrderedDict())
self.sources = defaultdict(list)
+ @property
+ def output_path(self):
+ if self.installed:
+ return ObjDirPath(
+ self._context, "!/" + mozpath.join(self.install_target, self.name)
+ )
+ else:
+ return ObjDirPath(self._context, "!" + self.name)
+
def link_library(self, obj):
assert isinstance(obj, BaseLibrary)
if obj.KIND != self.KIND:
@@ -481,15 +490,6 @@ class BaseProgram(Linkable):
self.program = program
self.is_unit_test = is_unit_test
- @property
- def output_path(self):
- if self.installed:
- return ObjDirPath(
- self._context, "!/" + mozpath.join(self.install_target, self.program)
- )
- else:
- return ObjDirPath(self._context, "!" + self.program)
-
def __repr__(self):
return "<%s: %s/%s>" % (type(self).__name__, self.relobjdir, self.program)
@@ -637,6 +637,10 @@ class BaseLibrary(Linkable):
def name(self):
return self.lib_name
+ @property
+ def import_path(self):
+ return mozpath.join(self.objdir, self.import_name)
+
class Library(BaseLibrary):
"""Context derived container object for a library"""
@@ -873,6 +877,16 @@ class SharedLibrary(Library):
elif context.config.substs.get("GCC_USE_GNU_LD"):
self.symbols_link_arg = "-Wl,--version-script," + self.symbols_file
+ @property
+ def import_path(self):
+ if self.config.substs.get("OS_ARCH") == "WINNT":
+ # We build import libs on windows in a library's objdir
+ # to avoid cluttering up dist/bin.
+ return mozpath.join(self.objdir, self.import_name)
+ return mozpath.join(
+ mozpath.dirname(self.output_path.full_path), self.import_name
+ )
+
class HostSharedLibrary(HostMixin, Library):
"""Context derived container object for a host shared library.
diff --git a/python/mozbuild/mozbuild/frontend/emitter.py b/python/mozbuild/mozbuild/frontend/emitter.py
index ac8a360bb1..635d695149 100644
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -17,7 +17,7 @@ from mach.mixin.logging import LoggingMixin
from mozpack.chrome.manifest import Manifest
from mozbuild.base import ExecutionSummary
-from mozbuild.util import OrderedDefaultDict, memoize
+from mozbuild.util import memoize
from ..testing import REFTEST_FLAVORS, TEST_MANIFESTS, SupportFilesConverter
from .context import Context, ObjDirPath, Path, SourcePath, SubContext
@@ -92,7 +92,7 @@ class TreeMetadataEmitter(LoggingMixin):
self.info = dict(mozinfo.info)
- self._libs = OrderedDefaultDict(list)
+ self._libs = defaultdict(list)
self._binaries = OrderedDict()
self._compile_dirs = set()
self._host_compile_dirs = set()
diff --git a/python/mozbuild/mozbuild/gn_processor.py b/python/mozbuild/mozbuild/gn_processor.py
index a77b6c7759..3a9b9e7f3b 100644
--- a/python/mozbuild/mozbuild/gn_processor.py
+++ b/python/mozbuild/mozbuild/gn_processor.py
@@ -17,8 +17,8 @@ import mozpack.path as mozpath
import six
from mozbuild.bootstrap import bootstrap_toolchain
+from mozbuild.dirutils import mkdir
from mozbuild.frontend.sandbox import alphabetical_sorted
-from mozbuild.util import mkdir
license_header = """# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
diff --git a/python/mozbuild/mozbuild/jar.py b/python/mozbuild/mozbuild/jar.py
index a595cee164..0559a12c29 100644
--- a/python/mozbuild/mozbuild/jar.py
+++ b/python/mozbuild/mozbuild/jar.py
@@ -17,10 +17,7 @@ import sys
from time import localtime
import mozpack.path as mozpath
-import six
-from mozpack.files import FileFinder
from MozZipFile import ZipFile
-from six import BytesIO
from mozbuild.action.buildlist import addEntriesToListFile
from mozbuild.preprocessor import Preprocessor
@@ -45,7 +42,7 @@ class ZipEntry(object):
def __init__(self, name, zipfile):
self._zipfile = zipfile
self._name = name
- self._inner = BytesIO()
+ self._inner = io.BytesIO()
def write(self, content):
"""Append the given content to this zip entry"""
@@ -341,7 +338,7 @@ class JarMaker(object):
myregister = dict.fromkeys(
map(lambda s: s.replace("%", chromebasepath), register)
)
- addEntriesToListFile(manifestPath, six.iterkeys(myregister))
+ addEntriesToListFile(manifestPath, myregister.keys())
def makeJar(self, infile, jardir):
"""makeJar is the main entry point to JarMaker.
@@ -360,7 +357,7 @@ class JarMaker(object):
self.localedirs = [_normpath(p) for p in self.localedirs]
elif self.relativesrcdir:
self.localedirs = self.generateLocaleDirs(self.relativesrcdir)
- if isinstance(infile, six.text_type):
+ if isinstance(infile, str):
logging.info("processing " + infile)
self.sourcedirs.append(_normpath(os.path.dirname(infile)))
pp = self.pp.clone()
@@ -467,6 +464,8 @@ class JarMaker(object):
prefix = "".join(_prefix(src))
emitted = set()
+ from mozpack.files import FileFinder
+
for _srcdir in src_base:
finder = FileFinder(_srcdir)
for path, _ in finder.find(src):
@@ -631,10 +630,7 @@ def main(args=None):
noise = logging.INFO
if options.verbose is not None:
noise = options.verbose and logging.DEBUG or logging.WARN
- if sys.version_info[:2] > (2, 3):
- logging.basicConfig(format="%(message)s")
- else:
- logging.basicConfig()
+ logging.basicConfig(format="%(message)s")
logging.getLogger().setLevel(noise)
topsrc = options.t
topsrc = os.path.normpath(os.path.abspath(topsrc))
@@ -642,5 +638,6 @@ def main(args=None):
infile = sys.stdin
else:
(infile,) = args
- infile = six.ensure_text(infile)
+ if isinstance(infile, bytes):
+ infile = infile.decode()
jm.makeJar(infile, options.d)
diff --git a/python/mozbuild/mozbuild/lock.py b/python/mozbuild/mozbuild/lock.py
new file mode 100644
index 0000000000..d3570358e1
--- /dev/null
+++ b/python/mozbuild/mozbuild/lock.py
@@ -0,0 +1,99 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This file contains miscellaneous utility functions that don't belong anywhere
+# in particular.
+
+import errno
+import os
+import stat
+import sys
+import time
+
+
+class LockFile(object):
+ """LockFile is used by the lock_file method to hold the lock.
+
+ This object should not be used directly, but only through
+ the lock_file method below.
+ """
+
+ def __init__(self, lockfile):
+ self.lockfile = lockfile
+
+ def __del__(self):
+ while True:
+ try:
+ os.remove(self.lockfile)
+ break
+ except OSError as e:
+ if e.errno == errno.EACCES:
+ # Another process probably has the file open, we'll retry.
+ # Just a short sleep since we want to drop the lock ASAP
+ # (but we need to let some other process close the file
+ # first).
+ time.sleep(0.1)
+ else:
+ # Re-raise unknown errors
+ raise
+
+
+def lock_file(lockfile, max_wait=600):
+ """Create and hold a lockfile of the given name, with the given timeout.
+
+ To release the lock, delete the returned object.
+ """
+
+ # FUTURE This function and object could be written as a context manager.
+
+ while True:
+ try:
+ fd = os.open(lockfile, os.O_EXCL | os.O_RDWR | os.O_CREAT)
+ # We created the lockfile, so we're the owner
+ break
+ except OSError as e:
+ if e.errno == errno.EEXIST or (
+ sys.platform == "win32" and e.errno == errno.EACCES
+ ):
+ pass
+ else:
+ # Should not occur
+ raise
+
+ try:
+ # The lock file exists, try to stat it to get its age
+ # and read its contents to report the owner PID
+ f = open(lockfile, "r")
+ s = os.stat(lockfile)
+ except EnvironmentError as e:
+ if e.errno == errno.ENOENT or e.errno == errno.EACCES:
+ # We didn't create the lockfile, so it did exist, but it's
+ # gone now. Just try again
+ continue
+
+ raise Exception(
+ "{0} exists but stat() failed: {1}".format(lockfile, e.strerror)
+ )
+
+ # We didn't create the lockfile and it's still there, check
+ # its age
+ now = int(time.time())
+ if now - s[stat.ST_MTIME] > max_wait:
+ pid = f.readline().rstrip()
+ raise Exception(
+ "{0} has been locked for more than "
+ "{1} seconds (PID {2})".format(lockfile, max_wait, pid)
+ )
+
+ # It's not been locked too long, wait a while and retry
+ f.close()
+ time.sleep(1)
+
+ # if we get here. we have the lockfile. Convert the os.open file
+ # descriptor into a Python file object and record our PID in it
+ f = os.fdopen(fd, "w")
+ f.write("{0}\n".format(os.getpid()))
+ f.close()
+
+ return LockFile(lockfile)
diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py
index 20b451dc16..01a0c7dc57 100644
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -1157,9 +1157,28 @@ def package(command_context, verbose=False):
)
if ret == 0:
command_context.notify("Packaging complete")
+ _print_package_name(command_context)
return ret
+def _print_package_name(command_context):
+ dist_path = mozpath.join(command_context.topobjdir, "dist")
+ package_name_path = mozpath.join(dist_path, "package_name.txt")
+ if not os.path.exists(package_name_path):
+ return
+
+ with open(package_name_path, "r") as f:
+ package_name = f.read().strip()
+ package_path = mozpath.join(dist_path, package_name)
+
+ if not os.path.exists(package_path):
+ return
+
+ command_context.log(
+ logging.INFO, "package", {}, "Created package: {}".format(package_path)
+ )
+
+
def _get_android_install_parser():
parser = argparse.ArgumentParser()
parser.add_argument(
diff --git a/python/mozbuild/mozbuild/makeutil.py b/python/mozbuild/mozbuild/makeutil.py
index 76691c5fa1..0f693c3f19 100644
--- a/python/mozbuild/mozbuild/makeutil.py
+++ b/python/mozbuild/mozbuild/makeutil.py
@@ -6,8 +6,6 @@ import os
import re
from collections.abc import Iterable
-import six
-
class Makefile(object):
"""Provides an interface for writing simple makefiles
@@ -26,7 +24,7 @@ class Makefile(object):
"""
targets = list(targets)
for target in targets:
- assert isinstance(target, six.text_type)
+ assert isinstance(target, str)
rule = Rule(targets)
self._statements.append(rule)
return rule
@@ -36,7 +34,7 @@ class Makefile(object):
Add a raw statement in the makefile. Meant to be used for
simple variable assignments.
"""
- assert isinstance(statement, six.text_type)
+ assert isinstance(statement, str)
self._statements.append(statement)
def dump(self, fh, removal_guard=True):
@@ -110,32 +108,28 @@ class Rule(object):
def add_targets(self, targets):
"""Add additional targets to the rule."""
- assert isinstance(targets, Iterable) and not isinstance(
- targets, six.string_types
- )
+ assert isinstance(targets, Iterable) and not isinstance(targets, str)
targets = list(targets)
for target in targets:
- assert isinstance(target, six.text_type)
+ assert isinstance(target, str)
self._targets.update(targets)
return self
def add_dependencies(self, deps):
"""Add dependencies to the rule."""
- assert isinstance(deps, Iterable) and not isinstance(deps, six.string_types)
+ assert isinstance(deps, Iterable) and not isinstance(deps, str)
deps = list(deps)
for dep in deps:
- assert isinstance(dep, six.text_type)
+ assert isinstance(dep, str)
self._dependencies.update(deps)
return self
def add_commands(self, commands):
"""Add commands to the rule."""
- assert isinstance(commands, Iterable) and not isinstance(
- commands, six.string_types
- )
+ assert isinstance(commands, Iterable) and not isinstance(commands, str)
commands = list(commands)
for command in commands:
- assert isinstance(command, six.text_type)
+ assert isinstance(command, str)
self._commands.extend(commands)
return self
@@ -180,7 +174,6 @@ def read_dep_makefile(fh):
rule = ""
for line in fh.readlines():
- line = six.ensure_text(line)
assert not line.startswith("\t")
line = line.strip()
if line.endswith("\\"):
diff --git a/python/mozbuild/mozbuild/mozconfig.py b/python/mozbuild/mozbuild/mozconfig.py
index 4322acbeed..bc3a849fd7 100644
--- a/python/mozbuild/mozbuild/mozconfig.py
+++ b/python/mozbuild/mozbuild/mozconfig.py
@@ -10,7 +10,6 @@ import traceback
from pathlib import Path
from textwrap import dedent
-import six
from mozboot.mozconfig import find_mozconfig
from mozpack import path as mozpath
@@ -57,7 +56,7 @@ class MozconfigLoadException(Exception):
{output}
"""
- ).format(output="\n".join([six.ensure_text(s) for s in self.output]))
+ ).format(output="\n".join(self.output))
Exception.__init__(self, message)
@@ -162,16 +161,15 @@ class MozconfigLoader(object):
try:
# We need to capture stderr because that's where the shell sends
# errors if execution fails.
- output = six.ensure_text(
- subprocess.check_output(
- command,
- stderr=subprocess.STDOUT,
- cwd=self.topsrcdir,
- env=env,
- universal_newlines=True,
- encoding="utf-8",
- )
+ output = subprocess.check_output(
+ command,
+ stderr=subprocess.STDOUT,
+ cwd=self.topsrcdir,
+ env=env,
+ universal_newlines=True,
+ encoding="utf-8",
)
+
except subprocess.CalledProcessError as e:
lines = e.output.splitlines()
diff --git a/python/mozbuild/mozbuild/repackaging/deb.py b/python/mozbuild/mozbuild/repackaging/deb.py
index 739fa5bfe4..b462402c7c 100644
--- a/python/mozbuild/mozbuild/repackaging/deb.py
+++ b/python/mozbuild/mozbuild/repackaging/deb.py
@@ -40,18 +40,33 @@ class HgServerError(Exception):
super().__init__(msg)
+# Maps our CI/release pipeline's architecture names (e.g., "x86_64")
+# into architectures ("amd64") compatible with Debian's dpkg-buildpackage tool.
+# This is the target architecture we are building the .deb package for.
_DEB_ARCH = {
"all": "all",
"x86": "i386",
"x86_64": "amd64",
+ "aarch64": "arm64",
+}
+
+# Defines the sysroot (build host's) architecture for each target architecture in the pipeline.
+# It defines the architecture dpkg-buildpackage runs on.
+_DEB_SYSROOT_ARCH = {
+ "all": "amd64",
+ "x86": "i386",
+ "x86_64": "amd64",
+ "aarch64": "amd64",
+}
+
+# Assigns the Debian distribution version for the sysroot based on the target architecture.
+# It defines the Debian distribution dpkg-buildpackage runs on.
+_DEB_SYSROOT_DIST = {
+ "all": "jessie",
+ "x86": "jessie",
+ "x86_64": "jessie",
+ "aarch64": "buster",
}
-# At the moment the Firefox build baseline is jessie.
-# The debian-repackage image defined in taskcluster/docker/debian-repackage/Dockerfile
-# bootstraps the /srv/jessie-i386 and /srv/jessie-amd64 chroot environments we use to
-# create the `.deb` repackages. By running the repackage using chroot we generate shared
-# library dependencies that match the Firefox build baseline
-# defined in taskcluster/scripts/misc/build-sysroot.sh
-_DEB_DIST = "jessie"
def repackage_deb(
@@ -729,8 +744,15 @@ def _is_chroot_available(arch):
def _get_chroot_path(arch):
- deb_arch = "amd64" if arch == "all" else _DEB_ARCH[arch]
- return f"/srv/{_DEB_DIST}-{deb_arch}"
+ # At the moment the Firefox build baseline for i386 and amd64 is jessie and the baseline for arm64 is buster.
+ # These baselines are defined in taskcluster/scripts/misc/build-sysroot.sh
+ # The debian-repackage image defined in taskcluster/docker/debian-repackage/Dockerfile
+ # bootstraps /srv/jessie-i386, /srv/jessie-amd64, and /srv/buster-amd64 roots.
+ # We use these roots to run the repackage step and generate shared
+ # library dependencies that match the Firefox build baseline.
+ deb_sysroot_dist = _DEB_SYSROOT_DIST[arch]
+ deb_sysroot_arch = _DEB_SYSROOT_ARCH[arch]
+ return f"/srv/{deb_sysroot_dist}-{deb_sysroot_arch}"
_MANIFEST_FILE_NAME = "manifest.json"
diff --git a/python/mozbuild/mozbuild/repackaging/installer.py b/python/mozbuild/mozbuild/repackaging/installer.py
index 9bd17613bf..bf2199ebb4 100644
--- a/python/mozbuild/mozbuild/repackaging/installer.py
+++ b/python/mozbuild/mozbuild/repackaging/installer.py
@@ -10,7 +10,7 @@ import zipfile
import mozpack.path as mozpath
from mozbuild.action.exe_7z_archive import archive_exe
-from mozbuild.util import ensureParentDir
+from mozbuild.dirutils import ensureParentDir
def repackage_installer(
diff --git a/python/mozbuild/mozbuild/repackaging/mar.py b/python/mozbuild/mozbuild/repackaging/mar.py
index f215c17238..46972398e8 100644
--- a/python/mozbuild/mozbuild/repackaging/mar.py
+++ b/python/mozbuild/mozbuild/repackaging/mar.py
@@ -14,8 +14,8 @@ from pathlib import Path
import mozfile
import mozpack.path as mozpath
+from mozbuild.dirutils import ensureParentDir
from mozbuild.repackaging.application_ini import get_application_ini_value
-from mozbuild.util import ensureParentDir
_BCJ_OPTIONS = {
"x86": ["--x86"],
diff --git a/python/mozbuild/mozbuild/repackaging/msi.py b/python/mozbuild/mozbuild/repackaging/msi.py
index 1884b05afe..3773ba14ae 100644
--- a/python/mozbuild/mozbuild/repackaging/msi.py
+++ b/python/mozbuild/mozbuild/repackaging/msi.py
@@ -11,7 +11,7 @@ from xml.dom import minidom
import mozpack.path as mozpath
-from mozbuild.util import ensureParentDir
+from mozbuild.dirutils import ensureParentDir
_MSI_ARCH = {
"x86": "x86",
diff --git a/python/mozbuild/mozbuild/repackaging/msix.py b/python/mozbuild/mozbuild/repackaging/msix.py
index 0836ffb87a..3165a9122b 100644
--- a/python/mozbuild/mozbuild/repackaging/msix.py
+++ b/python/mozbuild/mozbuild/repackaging/msix.py
@@ -33,8 +33,8 @@ from mozpack.mozjar import JarReader
from mozpack.packager.unpack import UnpackFinder
from six.moves import shlex_quote
+from mozbuild.dirutils import ensureParentDir
from mozbuild.repackaging.application_ini import get_application_ini_values
-from mozbuild.util import ensureParentDir
def log_copy_result(log, elapsed, destdir, result):
diff --git a/python/mozbuild/mozbuild/test/backend/common.py b/python/mozbuild/mozbuild/test/backend/common.py
index 07cfa7540f..7915b7a681 100644
--- a/python/mozbuild/mozbuild/test/backend/common.py
+++ b/python/mozbuild/mozbuild/test/backend/common.py
@@ -179,6 +179,15 @@ CONFIGS = defaultdict(
"BIN_SUFFIX": ".prog",
},
},
+ "shared-lib-paths": {
+ "defines": {},
+ "substs": {
+ "COMPILE_ENVIRONMENT": "1",
+ "LIB_SUFFIX": "a",
+ "DLL_PREFIX": "lib",
+ "DLL_SUFFIX": ".so",
+ },
+ },
"linkage": {
"defines": {},
"substs": {
diff --git a/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/dist-bin/moz.build b/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/dist-bin/moz.build
new file mode 100644
index 0000000000..2b2ef38337
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/dist-bin/moz.build
@@ -0,0 +1,4 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+SharedLibrary("dist-bin")
diff --git a/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/dist-subdir/moz.build b/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/dist-subdir/moz.build
new file mode 100644
index 0000000000..8d1bcf837a
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/dist-subdir/moz.build
@@ -0,0 +1,5 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+DIST_SUBDIR = "foo"
+SharedLibrary("dist-subdir")
diff --git a/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/final-target/moz.build b/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/final-target/moz.build
new file mode 100644
index 0000000000..2f58010ef4
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/final-target/moz.build
@@ -0,0 +1,5 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+FINAL_TARGET = "final/target"
+SharedLibrary("final-target")
diff --git a/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/moz.build b/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/moz.build
new file mode 100644
index 0000000000..4e3ed7cd0a
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/moz.build
@@ -0,0 +1,16 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+
+@template
+def SharedLibrary(name):
+ FORCE_SHARED_LIB = True
+ LIBRARY_NAME = name
+
+
+DIRS += [
+ "dist-bin",
+ "dist-subdir",
+ "final-target",
+ "not-installed",
+]
diff --git a/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/not-installed/moz.build b/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/not-installed/moz.build
new file mode 100644
index 0000000000..c0dfac027b
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/not-installed/moz.build
@@ -0,0 +1,5 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+DIST_INSTALL = False
+SharedLibrary("not-installed")
diff --git a/python/mozbuild/mozbuild/test/backend/test_build.py b/python/mozbuild/mozbuild/test/backend/test_build.py
index 3287ba5e57..571a4ae52d 100644
--- a/python/mozbuild/mozbuild/test/backend/test_build.py
+++ b/python/mozbuild/mozbuild/test/backend/test_build.py
@@ -21,9 +21,9 @@ from mozbuild.backend.configenvironment import ConfigEnvironment
from mozbuild.backend.fastermake import FasterMakeBackend
from mozbuild.backend.recursivemake import RecursiveMakeBackend
from mozbuild.base import MozbuildObject
+from mozbuild.dirutils import ensureParentDir
from mozbuild.frontend.emitter import TreeMetadataEmitter
from mozbuild.frontend.reader import BuildReader
-from mozbuild.util import ensureParentDir
def make_path():
diff --git a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
index 0706a302bf..a120ff81ce 100644
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -1203,23 +1203,23 @@ class TestRecursiveMakeBackend(BackendTester):
env = self._consume("linkage", RecursiveMakeBackend)
expected_linkage = {
"prog": {
- "SHARED_LIBS": ["qux/qux.so", "../shared/baz.so"],
+ "SHARED_LIBS": ["../dist/bin/qux.so", "../dist/bin/baz.so"],
"STATIC_LIBS": ["../real/foo.a"],
"OS_LIBS": ["-lfoo", "-lbaz", "-lbar"],
},
"shared": {
"OS_LIBS": ["-lfoo"],
- "SHARED_LIBS": ["../prog/qux/qux.so"],
+ "SHARED_LIBS": ["../dist/bin/qux.so"],
"STATIC_LIBS": [],
},
"static": {
"STATIC_LIBS": ["../real/foo.a"],
"OS_LIBS": ["-lbar"],
- "SHARED_LIBS": ["../prog/qux/qux.so"],
+ "SHARED_LIBS": ["../dist/bin/qux.so"],
},
"real": {
"STATIC_LIBS": [],
- "SHARED_LIBS": ["../prog/qux/qux.so"],
+ "SHARED_LIBS": ["../dist/bin/qux.so"],
"OS_LIBS": ["-lbaz"],
},
}
@@ -1326,6 +1326,28 @@ class TestRecursiveMakeBackend(BackendTester):
][0]
self.assertEqual(program, expected_program)
+ def test_shared_lib_paths(self):
+ """SHARED_LIBRARYs with various moz.build settings that change the destination should
+ produce the expected paths in backend.mk."""
+ env = self._consume("shared-lib-paths", RecursiveMakeBackend)
+
+ expected = [
+ ("dist-bin", "$(DEPTH)/dist/bin/libdist-bin.so"),
+ ("dist-subdir", "$(DEPTH)/dist/bin/foo/libdist-subdir.so"),
+ ("final-target", "$(DEPTH)/final/target/libfinal-target.so"),
+ ("not-installed", "libnot-installed.so"),
+ ]
+ prefix = "SHARED_LIBRARY := "
+ for subdir, expected_shared_lib in expected:
+ with io.open(os.path.join(env.topobjdir, subdir, "backend.mk"), "r") as fh:
+ lines = fh.readlines()
+ shared_lib = [
+ line.rstrip().split(prefix, 1)[1]
+ for line in lines
+ if line.startswith(prefix)
+ ][0]
+ self.assertEqual(shared_lib, expected_shared_lib)
+
if __name__ == "__main__":
main()
diff --git a/python/mozbuild/mozbuild/test/configure/test_checks_configure.py b/python/mozbuild/mozbuild/test/configure/test_checks_configure.py
index 131ac5aa7b..1388e4e31f 100644
--- a/python/mozbuild/mozbuild/test/configure/test_checks_configure.py
+++ b/python/mozbuild/mozbuild/test/configure/test_checks_configure.py
@@ -592,7 +592,7 @@ class TestChecksConfigure(unittest.TestCase):
javac = mozpath.abspath("/usr/bin/javac")
paths = {java: None, javac: None}
expected_error_message = (
- "ERROR: Could not locate Java at /mozbuild/jdk/jdk-17.0.10+7/bin, "
+ "ERROR: Could not locate Java at /mozbuild/jdk/jdk-17.0.11+9/bin, "
"please run ./mach bootstrap --no-system-changes\n"
)
diff --git a/python/mozbuild/mozbuild/test/configure/test_configure.py b/python/mozbuild/mozbuild/test/configure/test_configure.py
index a63ad9a15c..c46e324eee 100644
--- a/python/mozbuild/mozbuild/test/configure/test_configure.py
+++ b/python/mozbuild/mozbuild/test/configure/test_configure.py
@@ -14,6 +14,7 @@ from six import StringIO
from mozbuild.configure import ConfigureError, ConfigureSandbox
from mozbuild.configure.options import (
+ ConflictingOptionError,
InvalidOptionError,
NegativeOptionValue,
PositiveOptionValue,
@@ -1133,6 +1134,49 @@ class TestConfigure(unittest.TestCase):
self.assertEqual(str(e.exception), message)
+ def test_imply_option_conflict(self):
+ moz_configure = """
+ option('--with-foo', help='foo')
+ option('--with-env-foo', help='foo')
+ imply_option('--with-qux', True)
+ imply_option('QUX', "FOO", when='--with-env-foo')
+ imply_option('--with-qux', "FOO", when='--with-foo')
+ option('--with-qux', env="QUX", nargs='*', help='qux')
+ set_config('QUX', depends('--with-qux')(lambda x: x))
+ """
+ with self.assertRaises(ConflictingOptionError) as e:
+ with self.moz_configure(moz_configure):
+ self.get_config(["--with-foo"])
+
+ self.assertEqual(
+ str(e.exception),
+ "Cannot add '--with-qux=FOO' to the implied set because it conflicts"
+ " with '--with-qux' that was added earlier",
+ )
+
+ with self.assertRaises(InvalidOptionError) as e:
+ with self.moz_configure(moz_configure):
+ self.get_config(["--with-env-foo"])
+
+ config_path = mozpath.abspath(mozpath.join(test_data_path, "moz.configure"))
+
+ # TODO: the error message is weird.
+ self.assertEqual(
+ str(e.exception),
+ "'QUX=FOO' implied by 'imply_option at %s:5' conflicts "
+ "with '--with-qux' from the implied" % config_path,
+ )
+
+ with self.assertRaises(InvalidOptionError) as e:
+ with self.moz_configure(moz_configure):
+ self.get_config(["--with-foo", "--with-env-foo"])
+
+ self.assertEqual(
+ str(e.exception),
+ "Cannot add '--with-qux=FOO' to the implied set because it conflicts"
+ " with '--with-qux' that was added earlier",
+ )
+
def test_option_failures(self):
with self.assertRaises(ConfigureError) as e:
with self.moz_configure('option("--with-foo", help="foo")'):
diff --git a/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/dist-bin/moz.build b/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/dist-bin/moz.build
new file mode 100644
index 0000000000..2b2ef38337
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/dist-bin/moz.build
@@ -0,0 +1,4 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+SharedLibrary("dist-bin")
diff --git a/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/dist-subdir/moz.build b/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/dist-subdir/moz.build
new file mode 100644
index 0000000000..8d1bcf837a
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/dist-subdir/moz.build
@@ -0,0 +1,5 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+DIST_SUBDIR = "foo"
+SharedLibrary("dist-subdir")
diff --git a/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/final-target/moz.build b/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/final-target/moz.build
new file mode 100644
index 0000000000..2f58010ef4
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/final-target/moz.build
@@ -0,0 +1,5 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+FINAL_TARGET = "final/target"
+SharedLibrary("final-target")
diff --git a/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/moz.build b/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/moz.build
new file mode 100644
index 0000000000..4e3ed7cd0a
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/moz.build
@@ -0,0 +1,16 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+
+@template
+def SharedLibrary(name):
+ FORCE_SHARED_LIB = True
+ LIBRARY_NAME = name
+
+
+DIRS += [
+ "dist-bin",
+ "dist-subdir",
+ "final-target",
+ "not-installed",
+]
diff --git a/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/not-installed/moz.build b/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/not-installed/moz.build
new file mode 100644
index 0000000000..c0dfac027b
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/not-installed/moz.build
@@ -0,0 +1,5 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+DIST_INSTALL = False
+SharedLibrary("not-installed")
diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/mozbuild/mozbuild/test/frontend/test_emitter.py
index 13018ba5b2..e906635fdd 100644
--- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py
@@ -782,6 +782,22 @@ class TestEmitterBasic(unittest.TestCase):
],
)
+ def test_shared_lib_paths(self):
+ """Various moz.build settings that change the destination of SHARED_LIBRARY
+ should be accurately reflected in Program.output_path."""
+ reader = self.reader("shared-lib-paths")
+ objs = self.read_topsrcdir(reader)
+ prog_paths = [o.output_path for o in objs if isinstance(o, SharedLibrary)]
+ self.assertEqual(
+ prog_paths,
+ [
+ "!/dist/bin/libdist-bin.so",
+ "!/dist/bin/foo/libdist-subdir.so",
+ "!/final/target/libfinal-target.so",
+ "!libnot-installed.so",
+ ],
+ )
+
def test_host_program_paths(self):
"""The destination of a HOST_PROGRAM (almost always dist/host/bin)
should be accurately reflected in Program.output_path."""
diff --git a/python/mozbuild/mozbuild/test/test_containers.py b/python/mozbuild/mozbuild/test/test_containers.py
index 50dd0a4088..862c244f11 100644
--- a/python/mozbuild/mozbuild/test/test_containers.py
+++ b/python/mozbuild/mozbuild/test/test_containers.py
@@ -3,14 +3,13 @@
# You can obtain one at http://mozilla.org/MPL/2.0/.
import unittest
-from collections import OrderedDict
+from collections import OrderedDict, defaultdict
from mozunit import main
from mozbuild.util import (
KeyedDefaultDict,
List,
- OrderedDefaultDict,
ReadOnlyDefaultDict,
ReadOnlyDict,
ReadOnlyKeyedDefaultDict,
@@ -146,7 +145,7 @@ class TestOrderedDefaultDict(unittest.TestCase):
def test_simple(self):
original = OrderedDict(foo=1, bar=2)
- test = OrderedDefaultDict(bool, original)
+ test = defaultdict(bool, original)
self.assertEqual(original, test)
@@ -155,7 +154,7 @@ class TestOrderedDefaultDict(unittest.TestCase):
self.assertEqual(list(test), ["foo", "bar"])
def test_defaults(self):
- test = OrderedDefaultDict(bool, {"foo": 1})
+ test = defaultdict(bool, {"foo": 1})
self.assertEqual(test["foo"], 1)
diff --git a/python/mozbuild/mozbuild/toolchains.py b/python/mozbuild/mozbuild/toolchains.py
index c5418089bb..ca6b77852c 100644
--- a/python/mozbuild/mozbuild/toolchains.py
+++ b/python/mozbuild/mozbuild/toolchains.py
@@ -14,9 +14,7 @@ def toolchain_task_definitions():
# Don't import globally to allow this module being imported without
# the taskgraph module being available (e.g. standalone js)
params = {"level": os.environ.get("MOZ_SCM_LEVEL", "3")}
- root_dir = os.path.join(
- os.path.dirname(__file__), "..", "..", "..", "taskcluster", "ci"
- )
+ root_dir = os.path.join(os.path.dirname(__file__), "..", "..", "..", "taskcluster")
toolchains = load_tasks_for_kind(params, "toolchain", root_dir=root_dir)
aliased = {}
for t in toolchains.values():
diff --git a/python/mozbuild/mozbuild/util.py b/python/mozbuild/mozbuild/util.py
index 576bbe3f43..b8d1603bac 100644
--- a/python/mozbuild/mozbuild/util.py
+++ b/python/mozbuild/mozbuild/util.py
@@ -4,37 +4,29 @@
# This file contains miscellaneous utility functions that don't belong anywhere
# in particular.
-
import argparse
import collections
import collections.abc
import copy
-import ctypes
import difflib
-import errno
import functools
import hashlib
import io
import itertools
import os
import re
-import stat
import sys
-import time
-from collections import OrderedDict
from io import BytesIO, StringIO
-from pathlib import Path
import six
-from packaging.version import Version
+
+from mozbuild.dirutils import ensureParentDir
MOZBUILD_METRICS_PATH = os.path.abspath(
os.path.join(__file__, "..", "..", "metrics.yaml")
)
if sys.platform == "win32":
- _kernel32 = ctypes.windll.kernel32
- _FILE_ATTRIBUTE_NOT_CONTENT_INDEXED = 0x2000
system_encoding = "mbcs"
else:
system_encoding = "utf-8"
@@ -155,42 +147,6 @@ class ReadOnlyDefaultDict(ReadOnlyDict):
return value
-def ensureParentDir(path):
- """Ensures the directory parent to the given file exists."""
- d = os.path.dirname(path)
- if d and not os.path.exists(path):
- try:
- os.makedirs(d)
- except OSError as error:
- if error.errno != errno.EEXIST:
- raise
-
-
-def mkdir(path, not_indexed=False):
- """Ensure a directory exists.
-
- If ``not_indexed`` is True, an attribute is set that disables content
- indexing on the directory.
- """
- try:
- os.makedirs(path)
- except OSError as e:
- if e.errno != errno.EEXIST:
- raise
-
- if not_indexed:
- if sys.platform == "win32":
- if isinstance(path, six.string_types):
- fn = _kernel32.SetFileAttributesW
- else:
- fn = _kernel32.SetFileAttributesA
-
- fn(path, _FILE_ATTRIBUTE_NOT_CONTENT_INDEXED)
- elif sys.platform == "darwin":
- with open(os.path.join(path, ".metadata_never_index"), "a"):
- pass
-
-
def simple_diff(filename, old_lines, new_lines):
"""Returns the diff between old_lines and new_lines, in unified diff form,
as a list of lines.
@@ -912,105 +868,6 @@ class HierarchicalStringList(object):
)
-class LockFile(object):
- """LockFile is used by the lock_file method to hold the lock.
-
- This object should not be used directly, but only through
- the lock_file method below.
- """
-
- def __init__(self, lockfile):
- self.lockfile = lockfile
-
- def __del__(self):
- while True:
- try:
- os.remove(self.lockfile)
- break
- except OSError as e:
- if e.errno == errno.EACCES:
- # Another process probably has the file open, we'll retry.
- # Just a short sleep since we want to drop the lock ASAP
- # (but we need to let some other process close the file
- # first).
- time.sleep(0.1)
- else:
- # Re-raise unknown errors
- raise
-
-
-def lock_file(lockfile, max_wait=600):
- """Create and hold a lockfile of the given name, with the given timeout.
-
- To release the lock, delete the returned object.
- """
-
- # FUTURE This function and object could be written as a context manager.
-
- while True:
- try:
- fd = os.open(lockfile, os.O_EXCL | os.O_RDWR | os.O_CREAT)
- # We created the lockfile, so we're the owner
- break
- except OSError as e:
- if e.errno == errno.EEXIST or (
- sys.platform == "win32" and e.errno == errno.EACCES
- ):
- pass
- else:
- # Should not occur
- raise
-
- try:
- # The lock file exists, try to stat it to get its age
- # and read its contents to report the owner PID
- f = open(lockfile, "r")
- s = os.stat(lockfile)
- except EnvironmentError as e:
- if e.errno == errno.ENOENT or e.errno == errno.EACCES:
- # We didn't create the lockfile, so it did exist, but it's
- # gone now. Just try again
- continue
-
- raise Exception(
- "{0} exists but stat() failed: {1}".format(lockfile, e.strerror)
- )
-
- # We didn't create the lockfile and it's still there, check
- # its age
- now = int(time.time())
- if now - s[stat.ST_MTIME] > max_wait:
- pid = f.readline().rstrip()
- raise Exception(
- "{0} has been locked for more than "
- "{1} seconds (PID {2})".format(lockfile, max_wait, pid)
- )
-
- # It's not been locked too long, wait a while and retry
- f.close()
- time.sleep(1)
-
- # if we get here. we have the lockfile. Convert the os.open file
- # descriptor into a Python file object and record our PID in it
- f = os.fdopen(fd, "w")
- f.write("{0}\n".format(os.getpid()))
- f.close()
-
- return LockFile(lockfile)
-
-
-class OrderedDefaultDict(OrderedDict):
- """A combination of OrderedDict and defaultdict."""
-
- def __init__(self, default_factory, *args, **kwargs):
- OrderedDict.__init__(self, *args, **kwargs)
- self._default_factory = default_factory
-
- def __missing__(self, key):
- value = self[key] = self._default_factory()
- return value
-
-
class KeyedDefaultDict(dict):
"""Like a defaultdict, but the default_factory function takes the key as
argument"""
@@ -1341,13 +1198,6 @@ def ensure_unicode(value, encoding="utf-8"):
return value
-def process_time():
- if six.PY2:
- return time.clock()
- else:
- return time.process_time()
-
-
def hexdump(buf):
"""
Returns a list of hexdump-like lines corresponding to the given input buffer.
@@ -1377,18 +1227,3 @@ def hexdump(buf):
line += "|\n"
lines.append(line)
return lines
-
-
-def mozilla_build_version():
- mozilla_build = os.environ.get("MOZILLABUILD")
-
- version_file = Path(mozilla_build) / "VERSION"
-
- assert version_file.exists(), (
- f'The MozillaBuild VERSION file was not found at "{version_file}".\n'
- "Please check if MozillaBuild is installed correctly and that the"
- "`MOZILLABUILD` environment variable is to the correct path."
- )
-
- with version_file.open() as file:
- return Version(file.readline().rstrip("\n"))
diff --git a/python/mozbuild/mozbuild/vendor/rewrite_mozbuild.py b/python/mozbuild/mozbuild/vendor/rewrite_mozbuild.py
index 5434510bb0..1590267e34 100644
--- a/python/mozbuild/mozbuild/vendor/rewrite_mozbuild.py
+++ b/python/mozbuild/mozbuild/vendor/rewrite_mozbuild.py
@@ -219,14 +219,10 @@ import inspect
def node_to_name(code, node):
- if (
- not FORCE_DOWNGRADE_BEHAVIOR
- and sys.version_info[0] >= 3
- and sys.version_info[1] >= 8
- ):
- return ast.get_source_segment(code, node)
+ if FORCE_DOWNGRADE_BEHAVIOR:
+ return node.__class__.__name__
- return node.__class__.__name__
+ return ast.get_source_segment(code, node)
def get_attribute_label(node):
@@ -254,11 +250,7 @@ def get_attribute_label(node):
def ast_get_source_segment(code, node):
caller = inspect.stack()[1]
- if "sphinx" in caller.filename or (
- not FORCE_DOWNGRADE_BEHAVIOR
- and sys.version_info[0] >= 3
- and sys.version_info[1] >= 8
- ):
+ if "sphinx" in caller.filename or not FORCE_DOWNGRADE_BEHAVIOR:
return ast.original_get_source_segment(code, node)
if caller.function == "assignment_node_to_source_filename_list":
@@ -271,9 +263,8 @@ def ast_get_source_segment(code, node):
# Overwrite it so we don't accidently use it
-if sys.version_info[0] >= 3 and sys.version_info[1] >= 8:
- ast.original_get_source_segment = ast.get_source_segment
- ast.get_source_segment = ast_get_source_segment
+ast.original_get_source_segment = ast.get_source_segment
+ast.get_source_segment = ast_get_source_segment
##############################################
diff --git a/python/mozbuild/mozbuild/vendor/vendor_python.py b/python/mozbuild/mozbuild/vendor/vendor_python.py
index c969a3a157..d704e1ca5e 100644
--- a/python/mozbuild/mozbuild/vendor/vendor_python.py
+++ b/python/mozbuild/mozbuild/vendor/vendor_python.py
@@ -109,6 +109,11 @@ class VendorPython(MozbuildObject):
_copy_file_strip_carriage_return(lockfiles.pip_lockfile, requirements_out)
_copy_file_strip_carriage_return(lockfiles.poetry_lockfile, poetry_lockfile)
self.repository.add_remove_files(vendor_dir)
+ # explicitly add the content of the egg-info directory as it is
+ # covered by the hgignore pattern.
+ egg_info_files = list(vendor_dir.glob("**/*.egg-info/*"))
+ if egg_info_files:
+ self.repository.add_remove_files(*egg_info_files)
def _extract(self, src, dest, keep_extra_files=False):
"""extract source distribution into vendor directory"""
diff --git a/python/mozbuild/mozpack/dmg.py b/python/mozbuild/mozpack/dmg.py
index 4e094648fe..6be813524f 100644
--- a/python/mozbuild/mozpack/dmg.py
+++ b/python/mozbuild/mozpack/dmg.py
@@ -11,7 +11,7 @@ from typing import List
import mozfile
-from mozbuild.util import ensureParentDir
+from mozbuild.dirutils import ensureParentDir
is_linux = platform.system() == "Linux"
is_osx = platform.system() == "Darwin"
diff --git a/python/mozbuild/mozpack/test/test_files.py b/python/mozbuild/mozpack/test/test_files.py
index 1c86f2e0cc..cfa2cd8ffe 100644
--- a/python/mozbuild/mozpack/test/test_files.py
+++ b/python/mozbuild/mozpack/test/test_files.py
@@ -2,7 +2,8 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-from mozbuild.util import ensure_bytes, ensureParentDir
+from mozbuild.dirutils import ensureParentDir
+from mozbuild.util import ensure_bytes
from mozpack.errors import ErrorMessage, errors
from mozpack.files import (
AbsoluteSymlinkFile,
diff --git a/python/mozbuild/mozpack/test/test_unify.py b/python/mozbuild/mozpack/test/test_unify.py
index 15de50dccc..b2c3073f03 100644
--- a/python/mozbuild/mozpack/test/test_unify.py
+++ b/python/mozbuild/mozpack/test/test_unify.py
@@ -8,7 +8,7 @@ from io import StringIO
import mozunit
-from mozbuild.util import ensureParentDir
+from mozbuild.dirutils import ensureParentDir
from mozpack.errors import AccumulatedErrors, ErrorMessage, errors
from mozpack.files import FileFinder
from mozpack.mozjar import JarWriter