summaryrefslogtreecommitdiffstats
path: root/python/mozbuild/mozpack/manifests.py
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--python/mozbuild/mozpack/manifests.py483
1 files changed, 483 insertions, 0 deletions
diff --git a/python/mozbuild/mozpack/manifests.py b/python/mozbuild/mozpack/manifests.py
new file mode 100644
index 0000000000..2df6c729ea
--- /dev/null
+++ b/python/mozbuild/mozpack/manifests.py
@@ -0,0 +1,483 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+from contextlib import contextmanager
+
+import six
+
+import mozpack.path as mozpath
+
+from .files import (
+ AbsoluteSymlinkFile,
+ ExistingFile,
+ File,
+ FileFinder,
+ GeneratedFile,
+ HardlinkFile,
+ PreprocessedFile,
+)
+
+
+# This probably belongs in a more generic module. Where?
+@contextmanager
+def _auto_fileobj(path, fileobj, mode="r"):
+ if path and fileobj:
+ raise AssertionError("Only 1 of path or fileobj may be defined.")
+
+ if not path and not fileobj:
+ raise AssertionError("Must specified 1 of path or fileobj.")
+
+ if path:
+ fileobj = open(path, mode)
+
+ try:
+ yield fileobj
+ finally:
+ if path:
+ fileobj.close()
+
+
+class UnreadableInstallManifest(Exception):
+ """Raised when an invalid install manifest is parsed."""
+
+
+class InstallManifest(object):
+ """Describes actions to be used with a copier.FileCopier instance.
+
+ This class facilitates serialization and deserialization of data used to
+ construct a copier.FileCopier and to perform copy operations.
+
+ The manifest defines source paths, destination paths, and a mechanism by
+ which the destination file should come into existence.
+
+ Entries in the manifest correspond to the following types:
+
+ copy -- The file specified as the source path will be copied to the
+ destination path.
+
+ link -- The destination path will be a symlink or hardlink to the source
+ path. If symlinks are not supported, a copy will be performed.
+
+ exists -- The destination path is accounted for and won't be deleted by
+ the FileCopier. If the destination path doesn't exist, an error is
+ raised.
+
+ optional -- The destination path is accounted for and won't be deleted by
+ the FileCopier. No error is raised if the destination path does not
+ exist.
+
+ patternlink -- Paths matched by the expression in the source path
+ will be symlinked or hardlinked to the destination directory.
+
+ patterncopy -- Similar to patternlink except files are copied, not
+ symlinked/hardlinked.
+
+ preprocess -- The file specified at the source path will be run through
+ the preprocessor, and the output will be written to the destination
+ path.
+
+ content -- The destination file will be created with the given content.
+
+ Version 1 of the manifest was the initial version.
+ Version 2 added optional path support
+ Version 3 added support for pattern entries.
+ Version 4 added preprocessed file support.
+ Version 5 added content support.
+ """
+
+ CURRENT_VERSION = 5
+
+ FIELD_SEPARATOR = "\x1f"
+
+ # Negative values are reserved for non-actionable items, that is, metadata
+ # that doesn't describe files in the destination.
+ LINK = 1
+ COPY = 2
+ REQUIRED_EXISTS = 3
+ OPTIONAL_EXISTS = 4
+ PATTERN_LINK = 5
+ PATTERN_COPY = 6
+ PREPROCESS = 7
+ CONTENT = 8
+
+ def __init__(self, path=None, fileobj=None):
+ """Create a new InstallManifest entry.
+
+ If path is defined, the manifest will be populated with data from the
+ file path.
+
+ If fileobj is defined, the manifest will be populated with data read
+ from the specified file object.
+
+ Both path and fileobj cannot be defined.
+ """
+ self._dests = {}
+ self._source_files = set()
+
+ if path or fileobj:
+ with _auto_fileobj(path, fileobj, "r") as fh:
+ self._source_files.add(fh.name)
+ self._load_from_fileobj(fh)
+
+ def _load_from_fileobj(self, fileobj):
+ version = fileobj.readline().rstrip()
+ if version not in ("1", "2", "3", "4", "5"):
+ raise UnreadableInstallManifest("Unknown manifest version: %s" % version)
+
+ for line in fileobj:
+ # Explicitly strip on \n so we don't strip out the FIELD_SEPARATOR
+ # as well.
+ line = line.rstrip("\n")
+
+ fields = line.split(self.FIELD_SEPARATOR)
+
+ record_type = int(fields[0])
+
+ if record_type == self.LINK:
+ dest, source = fields[1:]
+ self.add_link(source, dest)
+ continue
+
+ if record_type == self.COPY:
+ dest, source = fields[1:]
+ self.add_copy(source, dest)
+ continue
+
+ if record_type == self.REQUIRED_EXISTS:
+ _, path = fields
+ self.add_required_exists(path)
+ continue
+
+ if record_type == self.OPTIONAL_EXISTS:
+ _, path = fields
+ self.add_optional_exists(path)
+ continue
+
+ if record_type == self.PATTERN_LINK:
+ _, base, pattern, dest = fields[1:]
+ self.add_pattern_link(base, pattern, dest)
+ continue
+
+ if record_type == self.PATTERN_COPY:
+ _, base, pattern, dest = fields[1:]
+ self.add_pattern_copy(base, pattern, dest)
+ continue
+
+ if record_type == self.PREPROCESS:
+ dest, source, deps, marker, defines, warnings = fields[1:]
+
+ self.add_preprocess(
+ source,
+ dest,
+ deps,
+ marker,
+ self._decode_field_entry(defines),
+ silence_missing_directive_warnings=bool(int(warnings)),
+ )
+ continue
+
+ if record_type == self.CONTENT:
+ dest, content = fields[1:]
+
+ self.add_content(
+ six.ensure_text(self._decode_field_entry(content)), dest
+ )
+ continue
+
+ # Don't fail for non-actionable items, allowing
+ # forward-compatibility with those we will add in the future.
+ if record_type >= 0:
+ raise UnreadableInstallManifest("Unknown record type: %d" % record_type)
+
+ def __len__(self):
+ return len(self._dests)
+
+ def __contains__(self, item):
+ return item in self._dests
+
+ def __eq__(self, other):
+ return isinstance(other, InstallManifest) and self._dests == other._dests
+
+ def __neq__(self, other):
+ return not self.__eq__(other)
+
+ def __ior__(self, other):
+ if not isinstance(other, InstallManifest):
+ raise ValueError("Can only | with another instance of InstallManifest.")
+
+ self.add_entries_from(other)
+
+ return self
+
+ def _encode_field_entry(self, data):
+ """Converts an object into a format that can be stored in the manifest file.
+
+ Complex data types, such as ``dict``, need to be converted into a text
+ representation before they can be written to a file.
+ """
+ return json.dumps(data, sort_keys=True)
+
+ def _decode_field_entry(self, data):
+ """Restores an object from a format that can be stored in the manifest file.
+
+ Complex data types, such as ``dict``, need to be converted into a text
+ representation before they can be written to a file.
+ """
+ return json.loads(data)
+
+ def write(self, path=None, fileobj=None, expand_pattern=False):
+ """Serialize this manifest to a file or file object.
+
+ If path is specified, that file will be written to. If fileobj is specified,
+ the serialized content will be written to that file object.
+
+ It is an error if both are specified.
+ """
+ with _auto_fileobj(path, fileobj, "wt") as fh:
+ fh.write("%d\n" % self.CURRENT_VERSION)
+
+ for dest in sorted(self._dests):
+ entry = self._dests[dest]
+
+ if expand_pattern and entry[0] in (
+ self.PATTERN_LINK,
+ self.PATTERN_COPY,
+ ):
+ type, base, pattern, dest = entry
+ type = self.LINK if type == self.PATTERN_LINK else self.COPY
+ finder = FileFinder(base)
+ paths = [f[0] for f in finder.find(pattern)]
+ for path in paths:
+ source = mozpath.join(base, path)
+ parts = ["%d" % type, mozpath.join(dest, path), source]
+ fh.write(
+ "%s\n"
+ % self.FIELD_SEPARATOR.join(
+ six.ensure_text(p) for p in parts
+ )
+ )
+ else:
+ parts = ["%d" % entry[0], dest]
+ parts.extend(entry[1:])
+ fh.write(
+ "%s\n"
+ % self.FIELD_SEPARATOR.join(six.ensure_text(p) for p in parts)
+ )
+
+ def add_link(self, source, dest):
+ """Add a link to this manifest.
+
+ dest will be either a symlink or hardlink to source.
+ """
+ self._add_entry(dest, (self.LINK, source))
+
+ def add_copy(self, source, dest):
+ """Add a copy to this manifest.
+
+ source will be copied to dest.
+ """
+ self._add_entry(dest, (self.COPY, source))
+
+ def add_required_exists(self, dest):
+ """Record that a destination file must exist.
+
+ This effectively prevents the listed file from being deleted.
+ """
+ self._add_entry(dest, (self.REQUIRED_EXISTS,))
+
+ def add_optional_exists(self, dest):
+ """Record that a destination file may exist.
+
+ This effectively prevents the listed file from being deleted. Unlike a
+ "required exists" file, files of this type do not raise errors if the
+ destination file does not exist.
+ """
+ self._add_entry(dest, (self.OPTIONAL_EXISTS,))
+
+ def add_pattern_link(self, base, pattern, dest):
+ """Add a pattern match that results in links being created.
+
+ A ``FileFinder`` will be created with its base set to ``base``
+ and ``FileFinder.find()`` will be called with ``pattern`` to discover
+ source files. Each source file will be either symlinked or hardlinked
+ under ``dest``.
+
+ Filenames under ``dest`` are constructed by taking the path fragment
+ after ``base`` and concatenating it with ``dest``. e.g.
+
+ <base>/foo/bar.h -> <dest>/foo/bar.h
+ """
+ self._add_entry(
+ mozpath.join(dest, pattern), (self.PATTERN_LINK, base, pattern, dest)
+ )
+
+ def add_pattern_copy(self, base, pattern, dest):
+ """Add a pattern match that results in copies.
+
+ See ``add_pattern_link()`` for usage.
+ """
+ self._add_entry(
+ mozpath.join(dest, pattern), (self.PATTERN_COPY, base, pattern, dest)
+ )
+
+ def add_preprocess(
+ self,
+ source,
+ dest,
+ deps,
+ marker="#",
+ defines={},
+ silence_missing_directive_warnings=False,
+ ):
+ """Add a preprocessed file to this manifest.
+
+ ``source`` will be passed through preprocessor.py, and the output will be
+ written to ``dest``.
+ """
+ self._add_entry(
+ dest,
+ (
+ self.PREPROCESS,
+ source,
+ deps,
+ marker,
+ self._encode_field_entry(defines),
+ "1" if silence_missing_directive_warnings else "0",
+ ),
+ )
+
+ def add_content(self, content, dest):
+ """Add a file with the given content."""
+ self._add_entry(
+ dest,
+ (
+ self.CONTENT,
+ self._encode_field_entry(content),
+ ),
+ )
+
+ def _add_entry(self, dest, entry):
+ if dest in self._dests:
+ raise ValueError("Item already in manifest: %s" % dest)
+
+ self._dests[dest] = entry
+
+ def add_entries_from(self, other, base=""):
+ """
+ Copy data from another mozpack.copier.InstallManifest
+ instance, adding an optional base prefix to the destination.
+
+ This allows to merge two manifests into a single manifest, or
+ two take the tagged union of two manifests.
+ """
+ # We must copy source files to ourselves so extra dependencies from
+ # the preprocessor are taken into account. Ideally, we would track
+ # which source file each entry came from. However, this is more
+ # complicated and not yet implemented. The current implementation
+ # will result in over invalidation, possibly leading to performance
+ # loss.
+ self._source_files |= other._source_files
+
+ for dest in sorted(other._dests):
+ new_dest = mozpath.join(base, dest) if base else dest
+ entry = other._dests[dest]
+ if entry[0] in (self.PATTERN_LINK, self.PATTERN_COPY):
+ entry_type, entry_base, entry_pattern, entry_dest = entry
+ new_entry_dest = mozpath.join(base, entry_dest) if base else entry_dest
+ new_entry = (entry_type, entry_base, entry_pattern, new_entry_dest)
+ else:
+ new_entry = tuple(entry)
+
+ self._add_entry(new_dest, new_entry)
+
+ def populate_registry(self, registry, defines_override={}, link_policy="symlink"):
+ """Populate a mozpack.copier.FileRegistry instance with data from us.
+
+ The caller supplied a FileRegistry instance (or at least something that
+ conforms to its interface) and that instance is populated with data
+ from this manifest.
+
+ Defines can be given to override the ones in the manifest for
+ preprocessing.
+
+ The caller can set a link policy. This determines whether symlinks,
+ hardlinks, or copies are used for LINK and PATTERN_LINK.
+ """
+ assert link_policy in ("symlink", "hardlink", "copy")
+ for dest in sorted(self._dests):
+ entry = self._dests[dest]
+ install_type = entry[0]
+
+ if install_type == self.LINK:
+ if link_policy == "symlink":
+ cls = AbsoluteSymlinkFile
+ elif link_policy == "hardlink":
+ cls = HardlinkFile
+ else:
+ cls = File
+ registry.add(dest, cls(entry[1]))
+ continue
+
+ if install_type == self.COPY:
+ registry.add(dest, File(entry[1]))
+ continue
+
+ if install_type == self.REQUIRED_EXISTS:
+ registry.add(dest, ExistingFile(required=True))
+ continue
+
+ if install_type == self.OPTIONAL_EXISTS:
+ registry.add(dest, ExistingFile(required=False))
+ continue
+
+ if install_type in (self.PATTERN_LINK, self.PATTERN_COPY):
+ _, base, pattern, dest = entry
+ finder = FileFinder(base)
+ paths = [f[0] for f in finder.find(pattern)]
+
+ if install_type == self.PATTERN_LINK:
+ if link_policy == "symlink":
+ cls = AbsoluteSymlinkFile
+ elif link_policy == "hardlink":
+ cls = HardlinkFile
+ else:
+ cls = File
+ else:
+ cls = File
+
+ for path in paths:
+ source = mozpath.join(base, path)
+ registry.add(mozpath.join(dest, path), cls(source))
+
+ continue
+
+ if install_type == self.PREPROCESS:
+ defines = self._decode_field_entry(entry[4])
+ if defines_override:
+ defines.update(defines_override)
+ registry.add(
+ dest,
+ PreprocessedFile(
+ entry[1],
+ depfile_path=entry[2],
+ marker=entry[3],
+ defines=defines,
+ extra_depends=self._source_files,
+ silence_missing_directive_warnings=bool(int(entry[5])),
+ ),
+ )
+
+ continue
+
+ if install_type == self.CONTENT:
+ # GeneratedFile expect the buffer interface, which the unicode
+ # type doesn't have, so encode to a str.
+ content = self._decode_field_entry(entry[1]).encode("utf-8")
+ registry.add(dest, GeneratedFile(content))
+ continue
+
+ raise Exception(
+ "Unknown install type defined in manifest: %d" % install_type
+ )