diff options
Diffstat (limited to 'python/mozbuild')
57 files changed, 637 insertions, 461 deletions
diff --git a/python/mozbuild/metrics.yaml b/python/mozbuild/metrics.yaml index 068dd6a389..5c81240302 100644 --- a/python/mozbuild/metrics.yaml +++ b/python/mozbuild/metrics.yaml @@ -22,7 +22,7 @@ mozbuild: - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 notification_emails: - build-telemetry@mozilla.com - - mhentges@mozilla.com + - ahochheiden@mozilla.com expires: never send_in_pings: - usage @@ -36,7 +36,7 @@ mozbuild: - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 notification_emails: - build-telemetry@mozilla.com - - mhentges@mozilla.com + - ahochheiden@mozilla.com expires: never send_in_pings: - usage @@ -50,7 +50,7 @@ mozbuild: - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 notification_emails: - build-telemetry@mozilla.com - - mhentges@mozilla.com + - ahochheiden@mozilla.com expires: never send_in_pings: - usage @@ -64,7 +64,7 @@ mozbuild: - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 notification_emails: - build-telemetry@mozilla.com - - mhentges@mozilla.com + - ahochheiden@mozilla.com expires: never send_in_pings: - usage @@ -78,7 +78,7 @@ mozbuild: - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 notification_emails: - build-telemetry@mozilla.com - - mhentges@mozilla.com + - ahochheiden@mozilla.com expires: never send_in_pings: - usage @@ -92,7 +92,7 @@ mozbuild: - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 notification_emails: - build-telemetry@mozilla.com - - mhentges@mozilla.com + - ahochheiden@mozilla.com expires: never send_in_pings: - usage @@ -106,7 +106,7 @@ mozbuild: - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34 notification_emails: - build-telemetry@mozilla.com - - mhentges@mozilla.com + - ahochheiden@mozilla.com expires: never send_in_pings: - usage @@ -120,7 +120,7 @@ mozbuild: - https://bugzilla.mozilla.org/show_bug.cgi?id=1526072#c15 notification_emails: - build-telemetry@mozilla.com - - mhentges@mozilla.com + - ahochheiden@mozilla.com expires: never send_in_pings: - usage @@ -134,7 +134,7 @@ mozbuild: - https://bugzilla.mozilla.org/show_bug.cgi?id=1654084#c2 notification_emails: - build-telemetry@mozilla.com - - mhentges@mozilla.com + - ahochheiden@mozilla.com expires: never send_in_pings: - usage diff --git a/python/mozbuild/mozbuild/action/buildlist.py b/python/mozbuild/mozbuild/action/buildlist.py index ed7149eb87..b0514053ce 100644 --- a/python/mozbuild/mozbuild/action/buildlist.py +++ b/python/mozbuild/mozbuild/action/buildlist.py @@ -11,7 +11,8 @@ import io import os import sys -from mozbuild.util import ensureParentDir, lock_file +from mozbuild.dirutils import ensureParentDir +from mozbuild.lock import lock_file def addEntriesToListFile(listFile, entries): diff --git a/python/mozbuild/mozbuild/action/dump_env.py b/python/mozbuild/mozbuild/action/dump_env.py index ec178700eb..de62089afd 100644 --- a/python/mozbuild/mozbuild/action/dump_env.py +++ b/python/mozbuild/mozbuild/action/dump_env.py @@ -12,19 +12,5 @@ sys.path.append(os.path.join(os.path.dirname(__file__), "..")) from shellutil import quote - -def environ(): - # We would use six.ensure_text but the global Python isn't guaranteed to have - # the correct version of six installed. - def ensure_text(s): - if sys.version_info > (3, 0) or isinstance(s, unicode): - # os.environ always returns string keys and values in Python 3. - return s - else: - return s.decode("utf-8") - - return [(ensure_text(k), ensure_text(v)) for (k, v) in os.environ.items()] - - -for key, value in environ(): +for key, value in os.environ.items(): print("%s=%s" % (key, quote(value))) diff --git a/python/mozbuild/mozbuild/action/l10n_merge.py b/python/mozbuild/mozbuild/action/l10n_merge.py index 1a04d60107..84975497fe 100644 --- a/python/mozbuild/mozbuild/action/l10n_merge.py +++ b/python/mozbuild/mozbuild/action/l10n_merge.py @@ -7,7 +7,7 @@ import os import shutil import sys -from mozbuild.util import ensureParentDir +from mozbuild.dirutils import ensureParentDir def main(argv): diff --git a/python/mozbuild/mozbuild/action/test_archive.py b/python/mozbuild/mozbuild/action/test_archive.py index 1e2fcc15c9..f5dd893d3b 100644 --- a/python/mozbuild/mozbuild/action/test_archive.py +++ b/python/mozbuild/mozbuild/action/test_archive.py @@ -24,7 +24,7 @@ from mozpack.manifests import InstallManifest from mozpack.mozjar import JarWriter from reftest import ReftestManifest -from mozbuild.util import ensureParentDir +from mozbuild.dirutils import ensureParentDir STAGE = mozpath.join(buildconfig.topobjdir, "dist", "test-stage") diff --git a/python/mozbuild/mozbuild/action/tooltool.py b/python/mozbuild/mozbuild/action/tooltool.py index 6b53db31e8..9abbff019e 100755 --- a/python/mozbuild/mozbuild/action/tooltool.py +++ b/python/mozbuild/mozbuild/action/tooltool.py @@ -61,25 +61,12 @@ REQUEST_HEADER_ATTRIBUTE_CHARS = re.compile( DEFAULT_MANIFEST_NAME = "manifest.tt" TOOLTOOL_PACKAGE_SUFFIX = ".TOOLTOOL-PACKAGE" HAWK_VER = 1 -PY3 = sys.version_info[0] == 3 - -if PY3: - six_binary_type = bytes - unicode = ( - str # Silence `pyflakes` from reporting `undefined name 'unicode'` in Python 3. - ) - import urllib.request as urllib2 - from http.client import HTTPConnection, HTTPSConnection - from urllib.error import HTTPError, URLError - from urllib.parse import urljoin, urlparse - from urllib.request import Request -else: - six_binary_type = str - import urllib2 - from httplib import HTTPConnection, HTTPSConnection - from urllib2 import HTTPError, Request, URLError - from urlparse import urljoin, urlparse +import urllib.request as urllib2 +from http.client import HTTPConnection, HTTPSConnection +from urllib.error import HTTPError, URLError +from urllib.parse import urljoin, urlparse +from urllib.request import Request log = logging.getLogger(__name__) @@ -205,9 +192,7 @@ def retriable(*retry_args, **retry_kwargs): def request_has_data(req): - if PY3: - return req.data is not None - return req.has_data() + return req.data is not None def get_hexdigest(val): @@ -281,7 +266,7 @@ def random_string(length): def prepare_header_val(val): - if isinstance(val, six_binary_type): + if isinstance(val, bytes): val = val.decode("utf-8") if not REQUEST_HEADER_ATTRIBUTE_CHARS.match(val): @@ -303,7 +288,7 @@ def parse_content_type(content_type): # pragma: no cover def calculate_payload_hash(algorithm, payload, content_type): # pragma: no cover parts = [ - part if isinstance(part, six_binary_type) else part.encode("utf8") + part if isinstance(part, bytes) else part.encode("utf8") for part in [ "hawk." + str(HAWK_VER) + ".payload\n", parse_content_type(content_type) + "\n", @@ -337,7 +322,7 @@ def validate_taskcluster_credentials(credentials): def normalize_header_attr(val): - if isinstance(val, six_binary_type): + if isinstance(val, bytes): return val.decode("utf-8") return val # pragma: no cover @@ -390,10 +375,10 @@ def calculate_mac( log.debug("normalized resource for mac calc: {norm}".format(norm=normalized)) digestmod = getattr(hashlib, algorithm) - if not isinstance(normalized, six_binary_type): + if not isinstance(normalized, bytes): normalized = normalized.encode("utf8") - if not isinstance(access_token, six_binary_type): + if not isinstance(access_token, bytes): access_token = access_token.encode("ascii") result = hmac.new(access_token, normalized, digestmod) @@ -412,10 +397,7 @@ def make_taskcluster_header(credentials, req): content_hash = None if request_has_data(req): - if PY3: - data = req.data - else: - data = req.get_data() + data = req.data content_hash = calculate_payload_hash( # pragma: no cover algorithm, data, @@ -760,7 +742,7 @@ def open_manifest(manifest_file): """I know how to take a filename and load it into a Manifest object""" if os.path.exists(manifest_file): manifest = Manifest() - with open(manifest_file, "r" if PY3 else "rb") as f: + with open(manifest_file, "r") as f: manifest.load(f) log.debug("loaded manifest from file '%s'" % manifest_file) return manifest @@ -865,12 +847,10 @@ def add_files(manifest_file, algorithm, filenames, version, visibility, unpack): for old_fr in old_manifest.file_records: if old_fr.filename not in new_filenames: new_manifest.file_records.append(old_fr) - if PY3: - with open(manifest_file, mode="w") as output: - new_manifest.dump(output, fmt="json") - else: - with open(manifest_file, mode="wb") as output: - new_manifest.dump(output, fmt="json") + + with open(manifest_file, mode="w") as output: + new_manifest.dump(output, fmt="json") + return all_files_added @@ -1288,9 +1268,7 @@ def _send_batch(base_url, auth_file, batch, region): url = urljoin(base_url, "upload") if region is not None: url += "?region=" + region - data = json.dumps(batch) - if PY3: - data = data.encode("utf-8") + data = json.dumps(batch).encode("utf-8") req = Request(url, data, {"Content-Type": "application/json"}) _authorize(req, auth_file) try: diff --git a/python/mozbuild/mozbuild/artifact_cache.py b/python/mozbuild/mozbuild/artifact_cache.py index 572953e1f7..bd6dd89697 100644 --- a/python/mozbuild/mozbuild/artifact_cache.py +++ b/python/mozbuild/mozbuild/artifact_cache.py @@ -31,7 +31,7 @@ import mozpack.path as mozpath import six import six.moves.urllib.parse as urlparse -from mozbuild.util import mkdir +from mozbuild.dirutils import mkdir # Using 'DownloadManager' through the provided interface we # can't directly specify a 'chunk_size' for the 'Download' it manages. diff --git a/python/mozbuild/mozbuild/artifact_commands.py b/python/mozbuild/mozbuild/artifact_commands.py index 62406f406a..e42b5a38d9 100644 --- a/python/mozbuild/mozbuild/artifact_commands.py +++ b/python/mozbuild/mozbuild/artifact_commands.py @@ -22,7 +22,7 @@ from mach.decorators import Command, CommandArgument, SubCommand from mozbuild.artifact_builds import JOB_CHOICES from mozbuild.base import MachCommandConditions as conditions -from mozbuild.util import ensureParentDir +from mozbuild.dirutils import ensureParentDir _COULD_NOT_FIND_ARTIFACTS_TEMPLATE = ( "ERROR!!!!!! Could not find artifacts for a toolchain build named " diff --git a/python/mozbuild/mozbuild/artifacts.py b/python/mozbuild/mozbuild/artifacts.py index eff16f6c9b..26ef99b1a2 100644 --- a/python/mozbuild/mozbuild/artifacts.py +++ b/python/mozbuild/mozbuild/artifacts.py @@ -65,7 +65,8 @@ from taskgraph.util.taskcluster import find_task_id, get_artifact_url, list_arti from mozbuild.artifact_builds import JOB_CHOICES from mozbuild.artifact_cache import ArtifactCache -from mozbuild.util import FileAvoidWrite, ensureParentDir, mkdir +from mozbuild.dirutils import ensureParentDir, mkdir +from mozbuild.util import FileAvoidWrite # Number of candidate pushheads to cache per parent changeset. NUM_PUSHHEADS_TO_QUERY_PER_PARENT = 50 @@ -151,14 +152,19 @@ class ArtifactJob(object): # prepended. # # The entries in the archive, suitably renamed, will be extracted into `dist`. - _extra_archives = { - ".xpt_artifacts.zip": { - "description": "XPT Artifacts", - "src_prefix": "", - "dest_prefix": "xpt_artifacts", - }, - } - _extra_archive_suffixes = tuple(sorted(_extra_archives.keys())) + @property + def _extra_archives(self): + return { + ".xpt_artifacts.zip": { + "description": "XPT Artifacts", + "src_prefix": "", + "dest_prefix": "xpt_artifacts", + }, + } + + @property + def _extra_archive_suffixes(self): + return tuple(sorted(self._extra_archives.keys())) def __init__( self, @@ -213,7 +219,7 @@ class ArtifactJob(object): self._symbols_archive_suffix ): yield name - elif name.endswith(ArtifactJob._extra_archive_suffixes): + elif name.endswith(self._extra_archive_suffixes): yield name else: self.log( @@ -247,7 +253,7 @@ class ArtifactJob(object): self._symbols_archive_suffix ): return self.process_symbols_archive(filename, processed_filename) - if filename.endswith(ArtifactJob._extra_archive_suffixes): + if filename.endswith(self._extra_archive_suffixes): return self.process_extra_archive(filename, processed_filename) return self.process_package_artifact(filename, processed_filename) @@ -401,7 +407,7 @@ class ArtifactJob(object): writer.add(destpath.encode("utf-8"), entry) def process_extra_archive(self, filename, processed_filename): - for suffix, extra_archive in ArtifactJob._extra_archives.items(): + for suffix, extra_archive in self._extra_archives.items(): if filename.endswith(suffix): self.log( logging.INFO, @@ -657,21 +663,28 @@ class MacArtifactJob(ArtifactJob): "nmhproxy", "pingsender", "plugin-container.app/Contents/MacOS/plugin-container", - "updater.app/Contents/Frameworks/UpdateSettings.framework/UpdateSettings", "updater.app/Contents/MacOS/org.mozilla.updater", # 'xpcshell', "XUL", ], ), - ( - "Contents/Frameworks", - [ - "ChannelPrefs.framework/ChannelPrefs", - ], - ), ) @property + def _extra_archives(self): + extra_archives = super()._extra_archives + extra_archives.update( + { + ".update_framework_artifacts.zip": { + "description": "Update-related macOS Framework Artifacts", + "src_prefix": "", + "dest_prefix": "update_framework_artifacts", + }, + } + ) + return extra_archives + + @property def paths_no_keep_path(self): formatted = [] for root, paths in self._paths_no_keep_path: @@ -1275,8 +1288,8 @@ class Artifacts(object): zeroes = "0" * 40 hashes = [] - for hg_hash in hg_hash_list.splitlines(): - hg_hash = hg_hash.strip() + for hg_hash_unstripped in hg_hash_list.splitlines(): + hg_hash = hg_hash_unstripped.strip() if not hg_hash or hg_hash == zeroes: continue hashes.append(hg_hash) @@ -1388,8 +1401,8 @@ https://firefox-source-docs.mozilla.org/contributing/vcs/mercurial_bundles.html if candidate_pushheads: break count = 0 - for rev in last_revs: - rev = rev.rstrip() + for rev_unstripped in last_revs: + rev = rev_unstripped.rstrip() if not rev: continue if rev not in candidate_pushheads: diff --git a/python/mozbuild/mozbuild/backend/common.py b/python/mozbuild/mozbuild/backend/common.py index b1c90f31fd..3e14484a30 100644 --- a/python/mozbuild/mozbuild/backend/common.py +++ b/python/mozbuild/mozbuild/backend/common.py @@ -13,6 +13,7 @@ import six from mozpack.chrome.manifest import parse_manifest_line from mozbuild.backend.base import BuildBackend +from mozbuild.dirutils import mkdir from mozbuild.frontend.context import ( VARIABLES, Context, @@ -44,7 +45,6 @@ from mozbuild.frontend.data import ( ) from mozbuild.jar import DeprecatedJarManifest, JarManifestParser from mozbuild.preprocessor import Preprocessor -from mozbuild.util import mkdir class XPIDLManager(object): @@ -182,7 +182,7 @@ class CommonBackend(BuildBackend): # the order is not consistent across multiple runs. # # Exclude this file in order to avoid breaking the - # taskcluster/ci/diffoscope/reproducible.yml jobs. + # taskcluster/kinds/diffoscope/reproducible.yml jobs. continue fullpath = ObjDirPath(obj._context, "!" + f).full_path self._handle_generated_sources([fullpath]) @@ -325,15 +325,13 @@ class CommonBackend(BuildBackend): list_style = "list" list_file_path = mozpath.join(objdir, name) objs = [os.path.relpath(o, objdir) for o in objs] - if list_style == "linkerscript": - ref = list_file_path - content = "\n".join('INPUT("%s")' % o for o in objs) - elif list_style == "filelist": + content = "\n".join(objs) + if list_style == "filelist": ref = "-Wl,-filelist," + list_file_path - content = "\n".join(objs) + elif list_style == "linkerlist": + ref = "-Wl,@" + list_file_path elif list_style == "list": ref = "@" + list_file_path - content = "\n".join(objs) else: return None diff --git a/python/mozbuild/mozbuild/backend/fastermake.py b/python/mozbuild/mozbuild/backend/fastermake.py index c423e00c32..3b0927defa 100644 --- a/python/mozbuild/mozbuild/backend/fastermake.py +++ b/python/mozbuild/mozbuild/backend/fastermake.py @@ -2,6 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +from collections import defaultdict from operator import itemgetter import mozpack.path as mozpath @@ -22,19 +23,18 @@ from mozbuild.frontend.data import ( XPIDLModule, ) from mozbuild.makeutil import Makefile -from mozbuild.util import OrderedDefaultDict class FasterMakeBackend(MakeBackend, PartialBackend): def _init(self): super(FasterMakeBackend, self)._init() - self._manifest_entries = OrderedDefaultDict(set) + self._manifest_entries = defaultdict(set) - self._install_manifests = OrderedDefaultDict(InstallManifest) + self._install_manifests = defaultdict(InstallManifest) - self._dependencies = OrderedDefaultDict(list) - self._l10n_dependencies = OrderedDefaultDict(list) + self._dependencies = defaultdict(list) + self._l10n_dependencies = defaultdict(list) self._has_xpidl = False diff --git a/python/mozbuild/mozbuild/backend/recursivemake.py b/python/mozbuild/mozbuild/backend/recursivemake.py index 86cb6ccc10..e3b5649d7e 100644 --- a/python/mozbuild/mozbuild/backend/recursivemake.py +++ b/python/mozbuild/mozbuild/backend/recursivemake.py @@ -69,7 +69,7 @@ from ..frontend.data import ( XPIDLModule, ) from ..makeutil import Makefile -from ..util import FileAvoidWrite, OrderedDefaultDict, ensureParentDir, pairwise +from ..util import FileAvoidWrite, ensureParentDir, pairwise from .common import CommonBackend from .make import MakeBackend @@ -364,7 +364,7 @@ class RecursiveMakeBackend(MakeBackend): self._install_manifests["dist_private"] self._traversal = RecursiveMakeTraversal() - self._compile_graph = OrderedDefaultDict(set) + self._compile_graph = defaultdict(set) self._rust_targets = set() self._gkrust_target = None self._pre_compile = set() @@ -810,7 +810,7 @@ class RecursiveMakeBackend(MakeBackend): rule.add_dependencies(sorted(deps)) non_default_roots = defaultdict(list) - non_default_graphs = defaultdict(lambda: OrderedDefaultDict(set)) + non_default_graphs = defaultdict(lambda: defaultdict(set)) for root in compile_roots: # If this is a non-default target, separate the root from the @@ -1366,7 +1366,8 @@ class RecursiveMakeBackend(MakeBackend): backend_file.write_once("LIBRARY_NAME := %s\n" % libdef.basename) backend_file.write("FORCE_SHARED_LIB := 1\n") backend_file.write("IMPORT_LIBRARY := %s\n" % libdef.import_name) - backend_file.write("SHARED_LIBRARY := %s\n" % libdef.lib_name) + shared_lib = self._pretty_path(libdef.output_path, backend_file) + backend_file.write("SHARED_LIBRARY := %s\n" % shared_lib) if libdef.soname: backend_file.write("DSO_SONAME := %s\n" % libdef.soname) if libdef.symbols_file: @@ -1375,11 +1376,7 @@ class RecursiveMakeBackend(MakeBackend): if not libdef.cxx_link: backend_file.write("LIB_IS_C_ONLY := 1\n") if libdef.output_category: - self._process_non_default_target(libdef, libdef.lib_name, backend_file) - # Override the install rule target for this library. This is hacky, - # but can go away as soon as we start building libraries in their - # final location (bug 1459764). - backend_file.write("SHARED_LIBRARY_TARGET := %s\n" % libdef.output_category) + self._process_non_default_target(libdef, shared_lib, backend_file) def _process_static_library(self, libdef, backend_file): backend_file.write_once("LIBRARY_NAME := %s\n" % libdef.basename) @@ -1427,15 +1424,13 @@ class RecursiveMakeBackend(MakeBackend): ) def _process_linked_libraries(self, obj, backend_file): - def pretty_relpath(lib, name): - return os.path.normpath( - mozpath.join(mozpath.relpath(lib.objdir, obj.objdir), name) - ) + def pretty_relpath(path): + return os.path.normpath(mozpath.relpath(path, obj.objdir)) objs, shared_libs, os_libs, static_libs = self._expand_libs(obj) obj_target = obj.name - if isinstance(obj, Program): + if isinstance(obj, (Program, SharedLibrary)): obj_target = self._pretty_path(obj.output_path, backend_file) objs_ref = " \\\n ".join(os.path.relpath(o, obj.objdir) for o in objs) @@ -1485,7 +1480,7 @@ class RecursiveMakeBackend(MakeBackend): for lib in shared_libs: assert obj.KIND != "host" and obj.KIND != "wasm" backend_file.write_once( - "SHARED_LIBS += %s\n" % pretty_relpath(lib, lib.import_name) + "SHARED_LIBS += %s\n" % pretty_relpath(lib.import_path) ) # We have to link any Rust libraries after all intermediate static @@ -1497,7 +1492,7 @@ class RecursiveMakeBackend(MakeBackend): (l for l in static_libs if isinstance(l, BaseRustLibrary)), ): backend_file.write_once( - "%s += %s\n" % (var, pretty_relpath(lib, lib.import_name)) + "%s += %s\n" % (var, pretty_relpath(lib.import_path)) ) for lib in os_libs: diff --git a/python/mozbuild/mozbuild/backend/visualstudio.py b/python/mozbuild/mozbuild/backend/visualstudio.py index ccf3c65a68..1b82bafbc8 100644 --- a/python/mozbuild/mozbuild/backend/visualstudio.py +++ b/python/mozbuild/mozbuild/backend/visualstudio.py @@ -8,7 +8,6 @@ import errno import os import re -import sys import uuid from pathlib import Path from xml.dom import getDOMImplementation @@ -35,8 +34,6 @@ MSNATVIS_NAMESPACE = "http://schemas.microsoft.com/vstudio/debugger/natvis/2010" def get_id(name): - if sys.version_info[0] == 2: - name = name.encode("utf-8") return str(uuid.uuid5(uuid.NAMESPACE_URL, name)).upper() diff --git a/python/mozbuild/mozbuild/buildversion.py b/python/mozbuild/mozbuild/buildversion.py new file mode 100644 index 0000000000..a0a767ab82 --- /dev/null +++ b/python/mozbuild/mozbuild/buildversion.py @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +from pathlib import Path + +from packaging.version import Version + + +def mozilla_build_version(): + mozilla_build = os.environ.get("MOZILLABUILD") + + version_file = Path(mozilla_build) / "VERSION" + + assert version_file.exists(), ( + f'The MozillaBuild VERSION file was not found at "{version_file}".\n' + "Please check if MozillaBuild is installed correctly and that the" + "`MOZILLABUILD` environment variable is to the correct path." + ) + + with version_file.open() as file: + return Version(file.readline().rstrip("\n")) diff --git a/python/mozbuild/mozbuild/config_status.py b/python/mozbuild/mozbuild/config_status.py index 8e8a7f625b..f99d0c5d3b 100644 --- a/python/mozbuild/mozbuild/config_status.py +++ b/python/mozbuild/mozbuild/config_status.py @@ -12,6 +12,8 @@ import sys import time from argparse import ArgumentParser from itertools import chain +from multiprocessing import Pool, get_start_method +from time import process_time from mach.logging import LoggingManager @@ -21,7 +23,7 @@ from mozbuild.base import MachCommandConditions from mozbuild.frontend.emitter import TreeMetadataEmitter from mozbuild.frontend.reader import BuildReader from mozbuild.mozinfo import write_mozinfo -from mozbuild.util import FileAvoidWrite, process_time +from mozbuild.util import FileAvoidWrite log_manager = LoggingManager() @@ -39,6 +41,45 @@ https://firefox-source-docs.mozilla.org/mobile/android/geckoview/contributor/gec """.strip() +## Parallel backend setup +# Distributing each backend on different process is costly because we need to +# copy the definitions across each process. These definitions are read-only, so +# only copy them once when each process starts. + + +class BackendPool: + per_process_definitions = None + + def __init__(self, definitions, *, processes=None): + definitions = list(definitions) + BackendPool._init_worker(definitions) + self.pool = Pool( + initializer=BackendPool._init_worker, + initargs=(definitions,), + processes=processes, + ) + + def run(self, backends): + # We're trying to spawn a minimal number of new processes there, and + # limit the number of times we serialize the task state. As a + # consequence: + # 1. we initialize each process with a copy of `definitions' + # 2. instead of spawning as many processes as backend, we use current + # process to handle one of the backend and asynchronously run the + # others. + async_tasks = self.pool.map_async(BackendPool._run_worker, backends[1:]) + BackendPool._run_worker(backends[0]) + async_tasks.wait() + + @staticmethod + def _init_worker(state): + BackendPool.per_process_definitions = state + + @staticmethod + def _run_worker(backend): + return backend.consume(BackendPool.per_process_definitions) + + def config_status( topobjdir=".", topsrcdir=".", @@ -148,11 +189,23 @@ def config_status( log_manager.enable_unstructured() print("Reticulating splines...", file=sys.stderr) - if len(selected_backends) > 1: - definitions = list(definitions) - for the_backend in selected_backends: - the_backend.consume(definitions) + # `definitions` objects are unfortunately not picklable, which is a + # requirement for "spawn" method. It's fine under "fork" method. This + # basically excludes Windows from our optimization, we can live with it. + if len(selected_backends) > 1 and get_start_method() == "fork": + # See https://github.com/python/cpython/commit/39889864c09741909da4ec489459d0197ea8f1fc + # For why we cap the process count. There's also an overhead to setup + # new processes, and not that many backends anyway. + processes = min(len(selected_backends) - 1, 4) + pool = BackendPool(definitions, processes=processes) + pool.run(selected_backends) + else: + if len(selected_backends) > 1: + definitions = list(definitions) + + for backend in selected_backends: + backend.consume(definitions) execution_time = 0.0 for obj in chain((reader, emitter), selected_backends): diff --git a/python/mozbuild/mozbuild/configure/lint.py b/python/mozbuild/mozbuild/configure/lint.py index c6bda9a540..4858188964 100644 --- a/python/mozbuild/mozbuild/configure/lint.py +++ b/python/mozbuild/mozbuild/configure/lint.py @@ -4,7 +4,6 @@ import inspect import re -import sys import types from dis import Bytecode from functools import wraps @@ -25,27 +24,9 @@ from .help import HelpFormatter def code_replace(code, co_filename, co_name, co_firstlineno): - if sys.version_info < (3, 8): - codetype_args = [ - code.co_argcount, - code.co_kwonlyargcount, - code.co_nlocals, - code.co_stacksize, - code.co_flags, - code.co_code, - code.co_consts, - code.co_names, - code.co_varnames, - co_filename, - co_name, - co_firstlineno, - code.co_lnotab, - ] - return types.CodeType(*codetype_args) - else: - return code.replace( - co_filename=co_filename, co_name=co_name, co_firstlineno=co_firstlineno - ) + return code.replace( + co_filename=co_filename, co_name=co_name, co_firstlineno=co_firstlineno + ) class LintSandbox(ConfigureSandbox): diff --git a/python/mozbuild/mozbuild/controller/building.py b/python/mozbuild/mozbuild/controller/building.py index 30f8136f26..75f8099672 100644 --- a/python/mozbuild/mozbuild/controller/building.py +++ b/python/mozbuild/mozbuild/controller/building.py @@ -34,9 +34,10 @@ from mozterm.widgets import Footer from ..backend import get_backend_class from ..base import MozbuildObject from ..compilation.warnings import WarningsCollector, WarningsDatabase +from ..dirutils import mkdir from ..telemetry import get_cpu_brand from ..testing import install_test_files -from ..util import FileAvoidWrite, mkdir, resolve_target_to_make +from ..util import FileAvoidWrite, resolve_target_to_make from .clobber import Clobberer FINDER_SLOW_MESSAGE = """ @@ -262,6 +263,12 @@ class BuildMonitor(MozbuildObject): _, _, disambiguator = args.pop(0).partition("@") action = args.pop(0) + time = None + regexp = re.compile(r"\d{10}(\.\d{1,9})?$") + if regexp.match(action): + time = float(action) + action = args.pop(0) + update_needed = True if action == "TIERS": @@ -279,12 +286,12 @@ class BuildMonitor(MozbuildObject): update_needed = False elif action.startswith("START_"): self.resources.begin_marker( - action[len("START_") :], " ".join(args), disambiguator + action[len("START_") :], " ".join(args), disambiguator, time ) update_needed = False elif action.startswith("END_"): self.resources.end_marker( - action[len("END_") :], " ".join(args), disambiguator + action[len("END_") :], " ".join(args), disambiguator, time ) update_needed = False elif action == "BUILD_VERBOSE": @@ -678,14 +685,14 @@ class BuildOutputManager(OutputManager): if message: self.log(logging.INFO, "build_output", {"line": message}, "{line}") elif state_changed: - have_handler = hasattr(self, "handler") + have_handler = hasattr(self, "_handler") if have_handler: - self.handler.acquire() + self._handler.acquire() try: self.refresh() finally: if have_handler: - self.handler.release() + self._handler.release() class StaticAnalysisFooter(Footer): @@ -740,14 +747,14 @@ class StaticAnalysisOutputManager(OutputManager): if relevant: self.log(logging.INFO, "build_output", {"line": line}, "{line}") else: - have_handler = hasattr(self, "handler") + have_handler = hasattr(self, "_handler") if have_handler: - self.handler.acquire() + self._handler.acquire() try: self.refresh() finally: if have_handler: - self.handler.release() + self._handler.release() def write(self, path, output_format): assert output_format in ("text", "json"), "Invalid output format {}".format( diff --git a/python/mozbuild/mozbuild/dirutils.py b/python/mozbuild/mozbuild/dirutils.py new file mode 100644 index 0000000000..efa4746301 --- /dev/null +++ b/python/mozbuild/mozbuild/dirutils.py @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file contains miscellaneous utility functions that don't belong anywhere +# in particular. + +import errno +import os +import sys + +if sys.platform == "win32": + import ctypes + + _kernel32 = ctypes.windll.kernel32 + _FILE_ATTRIBUTE_NOT_CONTENT_INDEXED = 0x2000 + + +def ensureParentDir(path): + """Ensures the directory parent to the given file exists.""" + d = os.path.dirname(path) + if d and not os.path.exists(path): + try: + os.makedirs(d) + except OSError as error: + if error.errno != errno.EEXIST: + raise + + +def mkdir(path, not_indexed=False): + """Ensure a directory exists. + + If ``not_indexed`` is True, an attribute is set that disables content + indexing on the directory. + """ + try: + os.makedirs(path) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + if not_indexed: + if sys.platform == "win32": + if isinstance(path, str): + fn = _kernel32.SetFileAttributesW + else: + fn = _kernel32.SetFileAttributesA + + fn(path, _FILE_ATTRIBUTE_NOT_CONTENT_INDEXED) + elif sys.platform == "darwin": + with open(os.path.join(path, ".metadata_never_index"), "a"): + pass diff --git a/python/mozbuild/mozbuild/doctor.py b/python/mozbuild/mozbuild/doctor.py index 315a00e7c0..2bcb91042e 100644 --- a/python/mozbuild/mozbuild/doctor.py +++ b/python/mozbuild/mozbuild/doctor.py @@ -3,7 +3,6 @@ # file, # You can obtain one at http://mozilla.org/MPL/2.0/. import enum -import locale import os import socket import subprocess @@ -460,39 +459,6 @@ def mozillabuild(**kwargs) -> DoctorCheck: @check -def bad_locale_utf8(**kwargs) -> DoctorCheck: - """Check to detect the invalid locale `UTF-8` on pre-3.8 Python.""" - if sys.version_info >= (3, 8): - return DoctorCheck( - name="utf8 locale", - status=CheckStatus.SKIPPED, - display_text=["Python version has fixed utf-8 locale bug."], - ) - - try: - # This line will attempt to get and parse the locale. - locale.getdefaultlocale() - - return DoctorCheck( - name="utf8 locale", - status=CheckStatus.OK, - display_text=["Python's locale is set to a valid value."], - ) - except ValueError: - return DoctorCheck( - name="utf8 locale", - status=CheckStatus.FATAL, - display_text=[ - "Your Python is using an invalid value for its locale.", - "Either update Python to version 3.8+, or set the following variables in ", - "your environment:", - " export LC_ALL=en_US.UTF-8", - " export LANG=en_US.UTF-8", - ], - ) - - -@check def artifact_build( topsrcdir: str, configure_args: Optional[List[str]], **kwargs ) -> DoctorCheck: diff --git a/python/mozbuild/mozbuild/dotproperties.py b/python/mozbuild/mozbuild/dotproperties.py index c02e1a794b..e65ecd8301 100644 --- a/python/mozbuild/mozbuild/dotproperties.py +++ b/python/mozbuild/mozbuild/dotproperties.py @@ -6,15 +6,9 @@ import codecs import re -import sys import six -if sys.version_info[0] == 3: - str_type = str -else: - str_type = basestring - class DotProperties: r"""A thin representation of a key=value .properties file.""" @@ -29,7 +23,7 @@ class DotProperties: Ignores empty lines and comment lines.""" - if isinstance(file, str_type): + if isinstance(file, str): f = codecs.open(file, "r", "utf-8") else: f = file diff --git a/python/mozbuild/mozbuild/frontend/data.py b/python/mozbuild/mozbuild/frontend/data.py index 4217196400..b958680aa5 100644 --- a/python/mozbuild/mozbuild/frontend/data.py +++ b/python/mozbuild/mozbuild/frontend/data.py @@ -391,6 +391,15 @@ class Linkable(ContextDerived): self.lib_defines = Defines(context, OrderedDict()) self.sources = defaultdict(list) + @property + def output_path(self): + if self.installed: + return ObjDirPath( + self._context, "!/" + mozpath.join(self.install_target, self.name) + ) + else: + return ObjDirPath(self._context, "!" + self.name) + def link_library(self, obj): assert isinstance(obj, BaseLibrary) if obj.KIND != self.KIND: @@ -481,15 +490,6 @@ class BaseProgram(Linkable): self.program = program self.is_unit_test = is_unit_test - @property - def output_path(self): - if self.installed: - return ObjDirPath( - self._context, "!/" + mozpath.join(self.install_target, self.program) - ) - else: - return ObjDirPath(self._context, "!" + self.program) - def __repr__(self): return "<%s: %s/%s>" % (type(self).__name__, self.relobjdir, self.program) @@ -637,6 +637,10 @@ class BaseLibrary(Linkable): def name(self): return self.lib_name + @property + def import_path(self): + return mozpath.join(self.objdir, self.import_name) + class Library(BaseLibrary): """Context derived container object for a library""" @@ -873,6 +877,16 @@ class SharedLibrary(Library): elif context.config.substs.get("GCC_USE_GNU_LD"): self.symbols_link_arg = "-Wl,--version-script," + self.symbols_file + @property + def import_path(self): + if self.config.substs.get("OS_ARCH") == "WINNT": + # We build import libs on windows in a library's objdir + # to avoid cluttering up dist/bin. + return mozpath.join(self.objdir, self.import_name) + return mozpath.join( + mozpath.dirname(self.output_path.full_path), self.import_name + ) + class HostSharedLibrary(HostMixin, Library): """Context derived container object for a host shared library. diff --git a/python/mozbuild/mozbuild/frontend/emitter.py b/python/mozbuild/mozbuild/frontend/emitter.py index ac8a360bb1..635d695149 100644 --- a/python/mozbuild/mozbuild/frontend/emitter.py +++ b/python/mozbuild/mozbuild/frontend/emitter.py @@ -17,7 +17,7 @@ from mach.mixin.logging import LoggingMixin from mozpack.chrome.manifest import Manifest from mozbuild.base import ExecutionSummary -from mozbuild.util import OrderedDefaultDict, memoize +from mozbuild.util import memoize from ..testing import REFTEST_FLAVORS, TEST_MANIFESTS, SupportFilesConverter from .context import Context, ObjDirPath, Path, SourcePath, SubContext @@ -92,7 +92,7 @@ class TreeMetadataEmitter(LoggingMixin): self.info = dict(mozinfo.info) - self._libs = OrderedDefaultDict(list) + self._libs = defaultdict(list) self._binaries = OrderedDict() self._compile_dirs = set() self._host_compile_dirs = set() diff --git a/python/mozbuild/mozbuild/gn_processor.py b/python/mozbuild/mozbuild/gn_processor.py index a77b6c7759..3a9b9e7f3b 100644 --- a/python/mozbuild/mozbuild/gn_processor.py +++ b/python/mozbuild/mozbuild/gn_processor.py @@ -17,8 +17,8 @@ import mozpack.path as mozpath import six from mozbuild.bootstrap import bootstrap_toolchain +from mozbuild.dirutils import mkdir from mozbuild.frontend.sandbox import alphabetical_sorted -from mozbuild.util import mkdir license_header = """# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/python/mozbuild/mozbuild/jar.py b/python/mozbuild/mozbuild/jar.py index a595cee164..0559a12c29 100644 --- a/python/mozbuild/mozbuild/jar.py +++ b/python/mozbuild/mozbuild/jar.py @@ -17,10 +17,7 @@ import sys from time import localtime import mozpack.path as mozpath -import six -from mozpack.files import FileFinder from MozZipFile import ZipFile -from six import BytesIO from mozbuild.action.buildlist import addEntriesToListFile from mozbuild.preprocessor import Preprocessor @@ -45,7 +42,7 @@ class ZipEntry(object): def __init__(self, name, zipfile): self._zipfile = zipfile self._name = name - self._inner = BytesIO() + self._inner = io.BytesIO() def write(self, content): """Append the given content to this zip entry""" @@ -341,7 +338,7 @@ class JarMaker(object): myregister = dict.fromkeys( map(lambda s: s.replace("%", chromebasepath), register) ) - addEntriesToListFile(manifestPath, six.iterkeys(myregister)) + addEntriesToListFile(manifestPath, myregister.keys()) def makeJar(self, infile, jardir): """makeJar is the main entry point to JarMaker. @@ -360,7 +357,7 @@ class JarMaker(object): self.localedirs = [_normpath(p) for p in self.localedirs] elif self.relativesrcdir: self.localedirs = self.generateLocaleDirs(self.relativesrcdir) - if isinstance(infile, six.text_type): + if isinstance(infile, str): logging.info("processing " + infile) self.sourcedirs.append(_normpath(os.path.dirname(infile))) pp = self.pp.clone() @@ -467,6 +464,8 @@ class JarMaker(object): prefix = "".join(_prefix(src)) emitted = set() + from mozpack.files import FileFinder + for _srcdir in src_base: finder = FileFinder(_srcdir) for path, _ in finder.find(src): @@ -631,10 +630,7 @@ def main(args=None): noise = logging.INFO if options.verbose is not None: noise = options.verbose and logging.DEBUG or logging.WARN - if sys.version_info[:2] > (2, 3): - logging.basicConfig(format="%(message)s") - else: - logging.basicConfig() + logging.basicConfig(format="%(message)s") logging.getLogger().setLevel(noise) topsrc = options.t topsrc = os.path.normpath(os.path.abspath(topsrc)) @@ -642,5 +638,6 @@ def main(args=None): infile = sys.stdin else: (infile,) = args - infile = six.ensure_text(infile) + if isinstance(infile, bytes): + infile = infile.decode() jm.makeJar(infile, options.d) diff --git a/python/mozbuild/mozbuild/lock.py b/python/mozbuild/mozbuild/lock.py new file mode 100644 index 0000000000..d3570358e1 --- /dev/null +++ b/python/mozbuild/mozbuild/lock.py @@ -0,0 +1,99 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file contains miscellaneous utility functions that don't belong anywhere +# in particular. + +import errno +import os +import stat +import sys +import time + + +class LockFile(object): + """LockFile is used by the lock_file method to hold the lock. + + This object should not be used directly, but only through + the lock_file method below. + """ + + def __init__(self, lockfile): + self.lockfile = lockfile + + def __del__(self): + while True: + try: + os.remove(self.lockfile) + break + except OSError as e: + if e.errno == errno.EACCES: + # Another process probably has the file open, we'll retry. + # Just a short sleep since we want to drop the lock ASAP + # (but we need to let some other process close the file + # first). + time.sleep(0.1) + else: + # Re-raise unknown errors + raise + + +def lock_file(lockfile, max_wait=600): + """Create and hold a lockfile of the given name, with the given timeout. + + To release the lock, delete the returned object. + """ + + # FUTURE This function and object could be written as a context manager. + + while True: + try: + fd = os.open(lockfile, os.O_EXCL | os.O_RDWR | os.O_CREAT) + # We created the lockfile, so we're the owner + break + except OSError as e: + if e.errno == errno.EEXIST or ( + sys.platform == "win32" and e.errno == errno.EACCES + ): + pass + else: + # Should not occur + raise + + try: + # The lock file exists, try to stat it to get its age + # and read its contents to report the owner PID + f = open(lockfile, "r") + s = os.stat(lockfile) + except EnvironmentError as e: + if e.errno == errno.ENOENT or e.errno == errno.EACCES: + # We didn't create the lockfile, so it did exist, but it's + # gone now. Just try again + continue + + raise Exception( + "{0} exists but stat() failed: {1}".format(lockfile, e.strerror) + ) + + # We didn't create the lockfile and it's still there, check + # its age + now = int(time.time()) + if now - s[stat.ST_MTIME] > max_wait: + pid = f.readline().rstrip() + raise Exception( + "{0} has been locked for more than " + "{1} seconds (PID {2})".format(lockfile, max_wait, pid) + ) + + # It's not been locked too long, wait a while and retry + f.close() + time.sleep(1) + + # if we get here. we have the lockfile. Convert the os.open file + # descriptor into a Python file object and record our PID in it + f = os.fdopen(fd, "w") + f.write("{0}\n".format(os.getpid())) + f.close() + + return LockFile(lockfile) diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py index 20b451dc16..01a0c7dc57 100644 --- a/python/mozbuild/mozbuild/mach_commands.py +++ b/python/mozbuild/mozbuild/mach_commands.py @@ -1157,9 +1157,28 @@ def package(command_context, verbose=False): ) if ret == 0: command_context.notify("Packaging complete") + _print_package_name(command_context) return ret +def _print_package_name(command_context): + dist_path = mozpath.join(command_context.topobjdir, "dist") + package_name_path = mozpath.join(dist_path, "package_name.txt") + if not os.path.exists(package_name_path): + return + + with open(package_name_path, "r") as f: + package_name = f.read().strip() + package_path = mozpath.join(dist_path, package_name) + + if not os.path.exists(package_path): + return + + command_context.log( + logging.INFO, "package", {}, "Created package: {}".format(package_path) + ) + + def _get_android_install_parser(): parser = argparse.ArgumentParser() parser.add_argument( diff --git a/python/mozbuild/mozbuild/makeutil.py b/python/mozbuild/mozbuild/makeutil.py index 76691c5fa1..0f693c3f19 100644 --- a/python/mozbuild/mozbuild/makeutil.py +++ b/python/mozbuild/mozbuild/makeutil.py @@ -6,8 +6,6 @@ import os import re from collections.abc import Iterable -import six - class Makefile(object): """Provides an interface for writing simple makefiles @@ -26,7 +24,7 @@ class Makefile(object): """ targets = list(targets) for target in targets: - assert isinstance(target, six.text_type) + assert isinstance(target, str) rule = Rule(targets) self._statements.append(rule) return rule @@ -36,7 +34,7 @@ class Makefile(object): Add a raw statement in the makefile. Meant to be used for simple variable assignments. """ - assert isinstance(statement, six.text_type) + assert isinstance(statement, str) self._statements.append(statement) def dump(self, fh, removal_guard=True): @@ -110,32 +108,28 @@ class Rule(object): def add_targets(self, targets): """Add additional targets to the rule.""" - assert isinstance(targets, Iterable) and not isinstance( - targets, six.string_types - ) + assert isinstance(targets, Iterable) and not isinstance(targets, str) targets = list(targets) for target in targets: - assert isinstance(target, six.text_type) + assert isinstance(target, str) self._targets.update(targets) return self def add_dependencies(self, deps): """Add dependencies to the rule.""" - assert isinstance(deps, Iterable) and not isinstance(deps, six.string_types) + assert isinstance(deps, Iterable) and not isinstance(deps, str) deps = list(deps) for dep in deps: - assert isinstance(dep, six.text_type) + assert isinstance(dep, str) self._dependencies.update(deps) return self def add_commands(self, commands): """Add commands to the rule.""" - assert isinstance(commands, Iterable) and not isinstance( - commands, six.string_types - ) + assert isinstance(commands, Iterable) and not isinstance(commands, str) commands = list(commands) for command in commands: - assert isinstance(command, six.text_type) + assert isinstance(command, str) self._commands.extend(commands) return self @@ -180,7 +174,6 @@ def read_dep_makefile(fh): rule = "" for line in fh.readlines(): - line = six.ensure_text(line) assert not line.startswith("\t") line = line.strip() if line.endswith("\\"): diff --git a/python/mozbuild/mozbuild/mozconfig.py b/python/mozbuild/mozbuild/mozconfig.py index 4322acbeed..bc3a849fd7 100644 --- a/python/mozbuild/mozbuild/mozconfig.py +++ b/python/mozbuild/mozbuild/mozconfig.py @@ -10,7 +10,6 @@ import traceback from pathlib import Path from textwrap import dedent -import six from mozboot.mozconfig import find_mozconfig from mozpack import path as mozpath @@ -57,7 +56,7 @@ class MozconfigLoadException(Exception): {output} """ - ).format(output="\n".join([six.ensure_text(s) for s in self.output])) + ).format(output="\n".join(self.output)) Exception.__init__(self, message) @@ -162,16 +161,15 @@ class MozconfigLoader(object): try: # We need to capture stderr because that's where the shell sends # errors if execution fails. - output = six.ensure_text( - subprocess.check_output( - command, - stderr=subprocess.STDOUT, - cwd=self.topsrcdir, - env=env, - universal_newlines=True, - encoding="utf-8", - ) + output = subprocess.check_output( + command, + stderr=subprocess.STDOUT, + cwd=self.topsrcdir, + env=env, + universal_newlines=True, + encoding="utf-8", ) + except subprocess.CalledProcessError as e: lines = e.output.splitlines() diff --git a/python/mozbuild/mozbuild/repackaging/deb.py b/python/mozbuild/mozbuild/repackaging/deb.py index 739fa5bfe4..b462402c7c 100644 --- a/python/mozbuild/mozbuild/repackaging/deb.py +++ b/python/mozbuild/mozbuild/repackaging/deb.py @@ -40,18 +40,33 @@ class HgServerError(Exception): super().__init__(msg) +# Maps our CI/release pipeline's architecture names (e.g., "x86_64") +# into architectures ("amd64") compatible with Debian's dpkg-buildpackage tool. +# This is the target architecture we are building the .deb package for. _DEB_ARCH = { "all": "all", "x86": "i386", "x86_64": "amd64", + "aarch64": "arm64", +} + +# Defines the sysroot (build host's) architecture for each target architecture in the pipeline. +# It defines the architecture dpkg-buildpackage runs on. +_DEB_SYSROOT_ARCH = { + "all": "amd64", + "x86": "i386", + "x86_64": "amd64", + "aarch64": "amd64", +} + +# Assigns the Debian distribution version for the sysroot based on the target architecture. +# It defines the Debian distribution dpkg-buildpackage runs on. +_DEB_SYSROOT_DIST = { + "all": "jessie", + "x86": "jessie", + "x86_64": "jessie", + "aarch64": "buster", } -# At the moment the Firefox build baseline is jessie. -# The debian-repackage image defined in taskcluster/docker/debian-repackage/Dockerfile -# bootstraps the /srv/jessie-i386 and /srv/jessie-amd64 chroot environments we use to -# create the `.deb` repackages. By running the repackage using chroot we generate shared -# library dependencies that match the Firefox build baseline -# defined in taskcluster/scripts/misc/build-sysroot.sh -_DEB_DIST = "jessie" def repackage_deb( @@ -729,8 +744,15 @@ def _is_chroot_available(arch): def _get_chroot_path(arch): - deb_arch = "amd64" if arch == "all" else _DEB_ARCH[arch] - return f"/srv/{_DEB_DIST}-{deb_arch}" + # At the moment the Firefox build baseline for i386 and amd64 is jessie and the baseline for arm64 is buster. + # These baselines are defined in taskcluster/scripts/misc/build-sysroot.sh + # The debian-repackage image defined in taskcluster/docker/debian-repackage/Dockerfile + # bootstraps /srv/jessie-i386, /srv/jessie-amd64, and /srv/buster-amd64 roots. + # We use these roots to run the repackage step and generate shared + # library dependencies that match the Firefox build baseline. + deb_sysroot_dist = _DEB_SYSROOT_DIST[arch] + deb_sysroot_arch = _DEB_SYSROOT_ARCH[arch] + return f"/srv/{deb_sysroot_dist}-{deb_sysroot_arch}" _MANIFEST_FILE_NAME = "manifest.json" diff --git a/python/mozbuild/mozbuild/repackaging/installer.py b/python/mozbuild/mozbuild/repackaging/installer.py index 9bd17613bf..bf2199ebb4 100644 --- a/python/mozbuild/mozbuild/repackaging/installer.py +++ b/python/mozbuild/mozbuild/repackaging/installer.py @@ -10,7 +10,7 @@ import zipfile import mozpack.path as mozpath from mozbuild.action.exe_7z_archive import archive_exe -from mozbuild.util import ensureParentDir +from mozbuild.dirutils import ensureParentDir def repackage_installer( diff --git a/python/mozbuild/mozbuild/repackaging/mar.py b/python/mozbuild/mozbuild/repackaging/mar.py index f215c17238..46972398e8 100644 --- a/python/mozbuild/mozbuild/repackaging/mar.py +++ b/python/mozbuild/mozbuild/repackaging/mar.py @@ -14,8 +14,8 @@ from pathlib import Path import mozfile import mozpack.path as mozpath +from mozbuild.dirutils import ensureParentDir from mozbuild.repackaging.application_ini import get_application_ini_value -from mozbuild.util import ensureParentDir _BCJ_OPTIONS = { "x86": ["--x86"], diff --git a/python/mozbuild/mozbuild/repackaging/msi.py b/python/mozbuild/mozbuild/repackaging/msi.py index 1884b05afe..3773ba14ae 100644 --- a/python/mozbuild/mozbuild/repackaging/msi.py +++ b/python/mozbuild/mozbuild/repackaging/msi.py @@ -11,7 +11,7 @@ from xml.dom import minidom import mozpack.path as mozpath -from mozbuild.util import ensureParentDir +from mozbuild.dirutils import ensureParentDir _MSI_ARCH = { "x86": "x86", diff --git a/python/mozbuild/mozbuild/repackaging/msix.py b/python/mozbuild/mozbuild/repackaging/msix.py index 0836ffb87a..3165a9122b 100644 --- a/python/mozbuild/mozbuild/repackaging/msix.py +++ b/python/mozbuild/mozbuild/repackaging/msix.py @@ -33,8 +33,8 @@ from mozpack.mozjar import JarReader from mozpack.packager.unpack import UnpackFinder from six.moves import shlex_quote +from mozbuild.dirutils import ensureParentDir from mozbuild.repackaging.application_ini import get_application_ini_values -from mozbuild.util import ensureParentDir def log_copy_result(log, elapsed, destdir, result): diff --git a/python/mozbuild/mozbuild/test/backend/common.py b/python/mozbuild/mozbuild/test/backend/common.py index 07cfa7540f..7915b7a681 100644 --- a/python/mozbuild/mozbuild/test/backend/common.py +++ b/python/mozbuild/mozbuild/test/backend/common.py @@ -179,6 +179,15 @@ CONFIGS = defaultdict( "BIN_SUFFIX": ".prog", }, }, + "shared-lib-paths": { + "defines": {}, + "substs": { + "COMPILE_ENVIRONMENT": "1", + "LIB_SUFFIX": "a", + "DLL_PREFIX": "lib", + "DLL_SUFFIX": ".so", + }, + }, "linkage": { "defines": {}, "substs": { diff --git a/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/dist-bin/moz.build b/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/dist-bin/moz.build new file mode 100644 index 0000000000..2b2ef38337 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/dist-bin/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +SharedLibrary("dist-bin") diff --git a/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/dist-subdir/moz.build b/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/dist-subdir/moz.build new file mode 100644 index 0000000000..8d1bcf837a --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/dist-subdir/moz.build @@ -0,0 +1,5 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIST_SUBDIR = "foo" +SharedLibrary("dist-subdir") diff --git a/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/final-target/moz.build b/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/final-target/moz.build new file mode 100644 index 0000000000..2f58010ef4 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/final-target/moz.build @@ -0,0 +1,5 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +FINAL_TARGET = "final/target" +SharedLibrary("final-target") diff --git a/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/moz.build b/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/moz.build new file mode 100644 index 0000000000..4e3ed7cd0a --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/moz.build @@ -0,0 +1,16 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + + +@template +def SharedLibrary(name): + FORCE_SHARED_LIB = True + LIBRARY_NAME = name + + +DIRS += [ + "dist-bin", + "dist-subdir", + "final-target", + "not-installed", +] diff --git a/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/not-installed/moz.build b/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/not-installed/moz.build new file mode 100644 index 0000000000..c0dfac027b --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/shared-lib-paths/not-installed/moz.build @@ -0,0 +1,5 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIST_INSTALL = False +SharedLibrary("not-installed") diff --git a/python/mozbuild/mozbuild/test/backend/test_build.py b/python/mozbuild/mozbuild/test/backend/test_build.py index 3287ba5e57..571a4ae52d 100644 --- a/python/mozbuild/mozbuild/test/backend/test_build.py +++ b/python/mozbuild/mozbuild/test/backend/test_build.py @@ -21,9 +21,9 @@ from mozbuild.backend.configenvironment import ConfigEnvironment from mozbuild.backend.fastermake import FasterMakeBackend from mozbuild.backend.recursivemake import RecursiveMakeBackend from mozbuild.base import MozbuildObject +from mozbuild.dirutils import ensureParentDir from mozbuild.frontend.emitter import TreeMetadataEmitter from mozbuild.frontend.reader import BuildReader -from mozbuild.util import ensureParentDir def make_path(): diff --git a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py index 0706a302bf..a120ff81ce 100644 --- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py +++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py @@ -1203,23 +1203,23 @@ class TestRecursiveMakeBackend(BackendTester): env = self._consume("linkage", RecursiveMakeBackend) expected_linkage = { "prog": { - "SHARED_LIBS": ["qux/qux.so", "../shared/baz.so"], + "SHARED_LIBS": ["../dist/bin/qux.so", "../dist/bin/baz.so"], "STATIC_LIBS": ["../real/foo.a"], "OS_LIBS": ["-lfoo", "-lbaz", "-lbar"], }, "shared": { "OS_LIBS": ["-lfoo"], - "SHARED_LIBS": ["../prog/qux/qux.so"], + "SHARED_LIBS": ["../dist/bin/qux.so"], "STATIC_LIBS": [], }, "static": { "STATIC_LIBS": ["../real/foo.a"], "OS_LIBS": ["-lbar"], - "SHARED_LIBS": ["../prog/qux/qux.so"], + "SHARED_LIBS": ["../dist/bin/qux.so"], }, "real": { "STATIC_LIBS": [], - "SHARED_LIBS": ["../prog/qux/qux.so"], + "SHARED_LIBS": ["../dist/bin/qux.so"], "OS_LIBS": ["-lbaz"], }, } @@ -1326,6 +1326,28 @@ class TestRecursiveMakeBackend(BackendTester): ][0] self.assertEqual(program, expected_program) + def test_shared_lib_paths(self): + """SHARED_LIBRARYs with various moz.build settings that change the destination should + produce the expected paths in backend.mk.""" + env = self._consume("shared-lib-paths", RecursiveMakeBackend) + + expected = [ + ("dist-bin", "$(DEPTH)/dist/bin/libdist-bin.so"), + ("dist-subdir", "$(DEPTH)/dist/bin/foo/libdist-subdir.so"), + ("final-target", "$(DEPTH)/final/target/libfinal-target.so"), + ("not-installed", "libnot-installed.so"), + ] + prefix = "SHARED_LIBRARY := " + for subdir, expected_shared_lib in expected: + with io.open(os.path.join(env.topobjdir, subdir, "backend.mk"), "r") as fh: + lines = fh.readlines() + shared_lib = [ + line.rstrip().split(prefix, 1)[1] + for line in lines + if line.startswith(prefix) + ][0] + self.assertEqual(shared_lib, expected_shared_lib) + if __name__ == "__main__": main() diff --git a/python/mozbuild/mozbuild/test/configure/test_checks_configure.py b/python/mozbuild/mozbuild/test/configure/test_checks_configure.py index 131ac5aa7b..1388e4e31f 100644 --- a/python/mozbuild/mozbuild/test/configure/test_checks_configure.py +++ b/python/mozbuild/mozbuild/test/configure/test_checks_configure.py @@ -592,7 +592,7 @@ class TestChecksConfigure(unittest.TestCase): javac = mozpath.abspath("/usr/bin/javac") paths = {java: None, javac: None} expected_error_message = ( - "ERROR: Could not locate Java at /mozbuild/jdk/jdk-17.0.10+7/bin, " + "ERROR: Could not locate Java at /mozbuild/jdk/jdk-17.0.11+9/bin, " "please run ./mach bootstrap --no-system-changes\n" ) diff --git a/python/mozbuild/mozbuild/test/configure/test_configure.py b/python/mozbuild/mozbuild/test/configure/test_configure.py index a63ad9a15c..c46e324eee 100644 --- a/python/mozbuild/mozbuild/test/configure/test_configure.py +++ b/python/mozbuild/mozbuild/test/configure/test_configure.py @@ -14,6 +14,7 @@ from six import StringIO from mozbuild.configure import ConfigureError, ConfigureSandbox from mozbuild.configure.options import ( + ConflictingOptionError, InvalidOptionError, NegativeOptionValue, PositiveOptionValue, @@ -1133,6 +1134,49 @@ class TestConfigure(unittest.TestCase): self.assertEqual(str(e.exception), message) + def test_imply_option_conflict(self): + moz_configure = """ + option('--with-foo', help='foo') + option('--with-env-foo', help='foo') + imply_option('--with-qux', True) + imply_option('QUX', "FOO", when='--with-env-foo') + imply_option('--with-qux', "FOO", when='--with-foo') + option('--with-qux', env="QUX", nargs='*', help='qux') + set_config('QUX', depends('--with-qux')(lambda x: x)) + """ + with self.assertRaises(ConflictingOptionError) as e: + with self.moz_configure(moz_configure): + self.get_config(["--with-foo"]) + + self.assertEqual( + str(e.exception), + "Cannot add '--with-qux=FOO' to the implied set because it conflicts" + " with '--with-qux' that was added earlier", + ) + + with self.assertRaises(InvalidOptionError) as e: + with self.moz_configure(moz_configure): + self.get_config(["--with-env-foo"]) + + config_path = mozpath.abspath(mozpath.join(test_data_path, "moz.configure")) + + # TODO: the error message is weird. + self.assertEqual( + str(e.exception), + "'QUX=FOO' implied by 'imply_option at %s:5' conflicts " + "with '--with-qux' from the implied" % config_path, + ) + + with self.assertRaises(InvalidOptionError) as e: + with self.moz_configure(moz_configure): + self.get_config(["--with-foo", "--with-env-foo"]) + + self.assertEqual( + str(e.exception), + "Cannot add '--with-qux=FOO' to the implied set because it conflicts" + " with '--with-qux' that was added earlier", + ) + def test_option_failures(self): with self.assertRaises(ConfigureError) as e: with self.moz_configure('option("--with-foo", help="foo")'): diff --git a/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/dist-bin/moz.build b/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/dist-bin/moz.build new file mode 100644 index 0000000000..2b2ef38337 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/dist-bin/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +SharedLibrary("dist-bin") diff --git a/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/dist-subdir/moz.build b/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/dist-subdir/moz.build new file mode 100644 index 0000000000..8d1bcf837a --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/dist-subdir/moz.build @@ -0,0 +1,5 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIST_SUBDIR = "foo" +SharedLibrary("dist-subdir") diff --git a/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/final-target/moz.build b/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/final-target/moz.build new file mode 100644 index 0000000000..2f58010ef4 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/final-target/moz.build @@ -0,0 +1,5 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +FINAL_TARGET = "final/target" +SharedLibrary("final-target") diff --git a/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/moz.build b/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/moz.build new file mode 100644 index 0000000000..4e3ed7cd0a --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/moz.build @@ -0,0 +1,16 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + + +@template +def SharedLibrary(name): + FORCE_SHARED_LIB = True + LIBRARY_NAME = name + + +DIRS += [ + "dist-bin", + "dist-subdir", + "final-target", + "not-installed", +] diff --git a/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/not-installed/moz.build b/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/not-installed/moz.build new file mode 100644 index 0000000000..c0dfac027b --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/shared-lib-paths/not-installed/moz.build @@ -0,0 +1,5 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIST_INSTALL = False +SharedLibrary("not-installed") diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/mozbuild/mozbuild/test/frontend/test_emitter.py index 13018ba5b2..e906635fdd 100644 --- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py +++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py @@ -782,6 +782,22 @@ class TestEmitterBasic(unittest.TestCase): ], ) + def test_shared_lib_paths(self): + """Various moz.build settings that change the destination of SHARED_LIBRARY + should be accurately reflected in Program.output_path.""" + reader = self.reader("shared-lib-paths") + objs = self.read_topsrcdir(reader) + prog_paths = [o.output_path for o in objs if isinstance(o, SharedLibrary)] + self.assertEqual( + prog_paths, + [ + "!/dist/bin/libdist-bin.so", + "!/dist/bin/foo/libdist-subdir.so", + "!/final/target/libfinal-target.so", + "!libnot-installed.so", + ], + ) + def test_host_program_paths(self): """The destination of a HOST_PROGRAM (almost always dist/host/bin) should be accurately reflected in Program.output_path.""" diff --git a/python/mozbuild/mozbuild/test/test_containers.py b/python/mozbuild/mozbuild/test/test_containers.py index 50dd0a4088..862c244f11 100644 --- a/python/mozbuild/mozbuild/test/test_containers.py +++ b/python/mozbuild/mozbuild/test/test_containers.py @@ -3,14 +3,13 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. import unittest -from collections import OrderedDict +from collections import OrderedDict, defaultdict from mozunit import main from mozbuild.util import ( KeyedDefaultDict, List, - OrderedDefaultDict, ReadOnlyDefaultDict, ReadOnlyDict, ReadOnlyKeyedDefaultDict, @@ -146,7 +145,7 @@ class TestOrderedDefaultDict(unittest.TestCase): def test_simple(self): original = OrderedDict(foo=1, bar=2) - test = OrderedDefaultDict(bool, original) + test = defaultdict(bool, original) self.assertEqual(original, test) @@ -155,7 +154,7 @@ class TestOrderedDefaultDict(unittest.TestCase): self.assertEqual(list(test), ["foo", "bar"]) def test_defaults(self): - test = OrderedDefaultDict(bool, {"foo": 1}) + test = defaultdict(bool, {"foo": 1}) self.assertEqual(test["foo"], 1) diff --git a/python/mozbuild/mozbuild/toolchains.py b/python/mozbuild/mozbuild/toolchains.py index c5418089bb..ca6b77852c 100644 --- a/python/mozbuild/mozbuild/toolchains.py +++ b/python/mozbuild/mozbuild/toolchains.py @@ -14,9 +14,7 @@ def toolchain_task_definitions(): # Don't import globally to allow this module being imported without # the taskgraph module being available (e.g. standalone js) params = {"level": os.environ.get("MOZ_SCM_LEVEL", "3")} - root_dir = os.path.join( - os.path.dirname(__file__), "..", "..", "..", "taskcluster", "ci" - ) + root_dir = os.path.join(os.path.dirname(__file__), "..", "..", "..", "taskcluster") toolchains = load_tasks_for_kind(params, "toolchain", root_dir=root_dir) aliased = {} for t in toolchains.values(): diff --git a/python/mozbuild/mozbuild/util.py b/python/mozbuild/mozbuild/util.py index 576bbe3f43..b8d1603bac 100644 --- a/python/mozbuild/mozbuild/util.py +++ b/python/mozbuild/mozbuild/util.py @@ -4,37 +4,29 @@ # This file contains miscellaneous utility functions that don't belong anywhere # in particular. - import argparse import collections import collections.abc import copy -import ctypes import difflib -import errno import functools import hashlib import io import itertools import os import re -import stat import sys -import time -from collections import OrderedDict from io import BytesIO, StringIO -from pathlib import Path import six -from packaging.version import Version + +from mozbuild.dirutils import ensureParentDir MOZBUILD_METRICS_PATH = os.path.abspath( os.path.join(__file__, "..", "..", "metrics.yaml") ) if sys.platform == "win32": - _kernel32 = ctypes.windll.kernel32 - _FILE_ATTRIBUTE_NOT_CONTENT_INDEXED = 0x2000 system_encoding = "mbcs" else: system_encoding = "utf-8" @@ -155,42 +147,6 @@ class ReadOnlyDefaultDict(ReadOnlyDict): return value -def ensureParentDir(path): - """Ensures the directory parent to the given file exists.""" - d = os.path.dirname(path) - if d and not os.path.exists(path): - try: - os.makedirs(d) - except OSError as error: - if error.errno != errno.EEXIST: - raise - - -def mkdir(path, not_indexed=False): - """Ensure a directory exists. - - If ``not_indexed`` is True, an attribute is set that disables content - indexing on the directory. - """ - try: - os.makedirs(path) - except OSError as e: - if e.errno != errno.EEXIST: - raise - - if not_indexed: - if sys.platform == "win32": - if isinstance(path, six.string_types): - fn = _kernel32.SetFileAttributesW - else: - fn = _kernel32.SetFileAttributesA - - fn(path, _FILE_ATTRIBUTE_NOT_CONTENT_INDEXED) - elif sys.platform == "darwin": - with open(os.path.join(path, ".metadata_never_index"), "a"): - pass - - def simple_diff(filename, old_lines, new_lines): """Returns the diff between old_lines and new_lines, in unified diff form, as a list of lines. @@ -912,105 +868,6 @@ class HierarchicalStringList(object): ) -class LockFile(object): - """LockFile is used by the lock_file method to hold the lock. - - This object should not be used directly, but only through - the lock_file method below. - """ - - def __init__(self, lockfile): - self.lockfile = lockfile - - def __del__(self): - while True: - try: - os.remove(self.lockfile) - break - except OSError as e: - if e.errno == errno.EACCES: - # Another process probably has the file open, we'll retry. - # Just a short sleep since we want to drop the lock ASAP - # (but we need to let some other process close the file - # first). - time.sleep(0.1) - else: - # Re-raise unknown errors - raise - - -def lock_file(lockfile, max_wait=600): - """Create and hold a lockfile of the given name, with the given timeout. - - To release the lock, delete the returned object. - """ - - # FUTURE This function and object could be written as a context manager. - - while True: - try: - fd = os.open(lockfile, os.O_EXCL | os.O_RDWR | os.O_CREAT) - # We created the lockfile, so we're the owner - break - except OSError as e: - if e.errno == errno.EEXIST or ( - sys.platform == "win32" and e.errno == errno.EACCES - ): - pass - else: - # Should not occur - raise - - try: - # The lock file exists, try to stat it to get its age - # and read its contents to report the owner PID - f = open(lockfile, "r") - s = os.stat(lockfile) - except EnvironmentError as e: - if e.errno == errno.ENOENT or e.errno == errno.EACCES: - # We didn't create the lockfile, so it did exist, but it's - # gone now. Just try again - continue - - raise Exception( - "{0} exists but stat() failed: {1}".format(lockfile, e.strerror) - ) - - # We didn't create the lockfile and it's still there, check - # its age - now = int(time.time()) - if now - s[stat.ST_MTIME] > max_wait: - pid = f.readline().rstrip() - raise Exception( - "{0} has been locked for more than " - "{1} seconds (PID {2})".format(lockfile, max_wait, pid) - ) - - # It's not been locked too long, wait a while and retry - f.close() - time.sleep(1) - - # if we get here. we have the lockfile. Convert the os.open file - # descriptor into a Python file object and record our PID in it - f = os.fdopen(fd, "w") - f.write("{0}\n".format(os.getpid())) - f.close() - - return LockFile(lockfile) - - -class OrderedDefaultDict(OrderedDict): - """A combination of OrderedDict and defaultdict.""" - - def __init__(self, default_factory, *args, **kwargs): - OrderedDict.__init__(self, *args, **kwargs) - self._default_factory = default_factory - - def __missing__(self, key): - value = self[key] = self._default_factory() - return value - - class KeyedDefaultDict(dict): """Like a defaultdict, but the default_factory function takes the key as argument""" @@ -1341,13 +1198,6 @@ def ensure_unicode(value, encoding="utf-8"): return value -def process_time(): - if six.PY2: - return time.clock() - else: - return time.process_time() - - def hexdump(buf): """ Returns a list of hexdump-like lines corresponding to the given input buffer. @@ -1377,18 +1227,3 @@ def hexdump(buf): line += "|\n" lines.append(line) return lines - - -def mozilla_build_version(): - mozilla_build = os.environ.get("MOZILLABUILD") - - version_file = Path(mozilla_build) / "VERSION" - - assert version_file.exists(), ( - f'The MozillaBuild VERSION file was not found at "{version_file}".\n' - "Please check if MozillaBuild is installed correctly and that the" - "`MOZILLABUILD` environment variable is to the correct path." - ) - - with version_file.open() as file: - return Version(file.readline().rstrip("\n")) diff --git a/python/mozbuild/mozbuild/vendor/rewrite_mozbuild.py b/python/mozbuild/mozbuild/vendor/rewrite_mozbuild.py index 5434510bb0..1590267e34 100644 --- a/python/mozbuild/mozbuild/vendor/rewrite_mozbuild.py +++ b/python/mozbuild/mozbuild/vendor/rewrite_mozbuild.py @@ -219,14 +219,10 @@ import inspect def node_to_name(code, node): - if ( - not FORCE_DOWNGRADE_BEHAVIOR - and sys.version_info[0] >= 3 - and sys.version_info[1] >= 8 - ): - return ast.get_source_segment(code, node) + if FORCE_DOWNGRADE_BEHAVIOR: + return node.__class__.__name__ - return node.__class__.__name__ + return ast.get_source_segment(code, node) def get_attribute_label(node): @@ -254,11 +250,7 @@ def get_attribute_label(node): def ast_get_source_segment(code, node): caller = inspect.stack()[1] - if "sphinx" in caller.filename or ( - not FORCE_DOWNGRADE_BEHAVIOR - and sys.version_info[0] >= 3 - and sys.version_info[1] >= 8 - ): + if "sphinx" in caller.filename or not FORCE_DOWNGRADE_BEHAVIOR: return ast.original_get_source_segment(code, node) if caller.function == "assignment_node_to_source_filename_list": @@ -271,9 +263,8 @@ def ast_get_source_segment(code, node): # Overwrite it so we don't accidently use it -if sys.version_info[0] >= 3 and sys.version_info[1] >= 8: - ast.original_get_source_segment = ast.get_source_segment - ast.get_source_segment = ast_get_source_segment +ast.original_get_source_segment = ast.get_source_segment +ast.get_source_segment = ast_get_source_segment ############################################## diff --git a/python/mozbuild/mozbuild/vendor/vendor_python.py b/python/mozbuild/mozbuild/vendor/vendor_python.py index c969a3a157..d704e1ca5e 100644 --- a/python/mozbuild/mozbuild/vendor/vendor_python.py +++ b/python/mozbuild/mozbuild/vendor/vendor_python.py @@ -109,6 +109,11 @@ class VendorPython(MozbuildObject): _copy_file_strip_carriage_return(lockfiles.pip_lockfile, requirements_out) _copy_file_strip_carriage_return(lockfiles.poetry_lockfile, poetry_lockfile) self.repository.add_remove_files(vendor_dir) + # explicitly add the content of the egg-info directory as it is + # covered by the hgignore pattern. + egg_info_files = list(vendor_dir.glob("**/*.egg-info/*")) + if egg_info_files: + self.repository.add_remove_files(*egg_info_files) def _extract(self, src, dest, keep_extra_files=False): """extract source distribution into vendor directory""" diff --git a/python/mozbuild/mozpack/dmg.py b/python/mozbuild/mozpack/dmg.py index 4e094648fe..6be813524f 100644 --- a/python/mozbuild/mozpack/dmg.py +++ b/python/mozbuild/mozpack/dmg.py @@ -11,7 +11,7 @@ from typing import List import mozfile -from mozbuild.util import ensureParentDir +from mozbuild.dirutils import ensureParentDir is_linux = platform.system() == "Linux" is_osx = platform.system() == "Darwin" diff --git a/python/mozbuild/mozpack/test/test_files.py b/python/mozbuild/mozpack/test/test_files.py index 1c86f2e0cc..cfa2cd8ffe 100644 --- a/python/mozbuild/mozpack/test/test_files.py +++ b/python/mozbuild/mozpack/test/test_files.py @@ -2,7 +2,8 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from mozbuild.util import ensure_bytes, ensureParentDir +from mozbuild.dirutils import ensureParentDir +from mozbuild.util import ensure_bytes from mozpack.errors import ErrorMessage, errors from mozpack.files import ( AbsoluteSymlinkFile, diff --git a/python/mozbuild/mozpack/test/test_unify.py b/python/mozbuild/mozpack/test/test_unify.py index 15de50dccc..b2c3073f03 100644 --- a/python/mozbuild/mozpack/test/test_unify.py +++ b/python/mozbuild/mozpack/test/test_unify.py @@ -8,7 +8,7 @@ from io import StringIO import mozunit -from mozbuild.util import ensureParentDir +from mozbuild.dirutils import ensureParentDir from mozpack.errors import AccumulatedErrors, ErrorMessage, errors from mozpack.files import FileFinder from mozpack.mozjar import JarWriter |