Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 483 insertions, 0 deletions

diff --git a/python/mozbuild/mozpack/manifests.py b/python/mozbuild/mozpack/manifests.py
new file mode 100644
index 0000000000..2df6c729ea
--- /dev/null
+++ b/python/mozbuild/mozpack/manifests.py
@@ -0,0 +1,483 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+from contextlib import contextmanager
+
+import six
+
+import mozpack.path as mozpath
+
+from .files import (
+    AbsoluteSymlinkFile,
+    ExistingFile,
+    File,
+    FileFinder,
+    GeneratedFile,
+    HardlinkFile,
+    PreprocessedFile,
+)
+
+
+# This probably belongs in a more generic module. Where?
+@contextmanager
+def _auto_fileobj(path, fileobj, mode="r"):
+    if path and fileobj:
+        raise AssertionError("Only 1 of path or fileobj may be defined.")
+
+    if not path and not fileobj:
+        raise AssertionError("Must specified 1 of path or fileobj.")
+
+    if path:
+        fileobj = open(path, mode)
+
+    try:
+        yield fileobj
+    finally:
+        if path:
+            fileobj.close()
+
+
+class UnreadableInstallManifest(Exception):
+    """Raised when an invalid install manifest is parsed."""
+
+
+class InstallManifest(object):
+    """Describes actions to be used with a copier.FileCopier instance.
+
+    This class facilitates serialization and deserialization of data used to
+    construct a copier.FileCopier and to perform copy operations.
+
+    The manifest defines source paths, destination paths, and a mechanism by
+    which the destination file should come into existence.
+
+    Entries in the manifest correspond to the following types:
+
+      copy -- The file specified as the source path will be copied to the
+          destination path.
+
+      link -- The destination path will be a symlink or hardlink to the source
+          path. If symlinks are not supported, a copy will be performed.
+
+      exists -- The destination path is accounted for and won't be deleted by
+          the FileCopier. If the destination path doesn't exist, an error is
+          raised.
+
+      optional -- The destination path is accounted for and won't be deleted by
+          the FileCopier. No error is raised if the destination path does not
+          exist.
+
+      patternlink -- Paths matched by the expression in the source path
+          will be symlinked or hardlinked to the destination directory.
+
+      patterncopy -- Similar to patternlink except files are copied, not
+          symlinked/hardlinked.
+
+      preprocess -- The file specified at the source path will be run through
+          the preprocessor, and the output will be written to the destination
+          path.
+
+      content -- The destination file will be created with the given content.
+
+    Version 1 of the manifest was the initial version.
+    Version 2 added optional path support
+    Version 3 added support for pattern entries.
+    Version 4 added preprocessed file support.
+    Version 5 added content support.
+    """
+
+    CURRENT_VERSION = 5
+
+    FIELD_SEPARATOR = "\x1f"
+
+    # Negative values are reserved for non-actionable items, that is, metadata
+    # that doesn't describe files in the destination.
+    LINK = 1
+    COPY = 2
+    REQUIRED_EXISTS = 3
+    OPTIONAL_EXISTS = 4
+    PATTERN_LINK = 5
+    PATTERN_COPY = 6
+    PREPROCESS = 7
+    CONTENT = 8
+
+    def __init__(self, path=None, fileobj=None):
+        """Create a new InstallManifest entry.
+
+        If path is defined, the manifest will be populated with data from the
+        file path.
+
+        If fileobj is defined, the manifest will be populated with data read
+        from the specified file object.
+
+        Both path and fileobj cannot be defined.
+        """
+        self._dests = {}
+        self._source_files = set()
+
+        if path or fileobj:
+            with _auto_fileobj(path, fileobj, "r") as fh:
+                self._source_files.add(fh.name)
+                self._load_from_fileobj(fh)
+
+    def _load_from_fileobj(self, fileobj):
+        version = fileobj.readline().rstrip()
+        if version not in ("1", "2", "3", "4", "5"):
+            raise UnreadableInstallManifest("Unknown manifest version: %s" % version)
+
+        for line in fileobj:
+            # Explicitly strip on \n so we don't strip out the FIELD_SEPARATOR
+            # as well.
+            line = line.rstrip("\n")
+
+            fields = line.split(self.FIELD_SEPARATOR)
+
+            record_type = int(fields[0])
+
+            if record_type == self.LINK:
+                dest, source = fields[1:]
+                self.add_link(source, dest)
+                continue
+
+            if record_type == self.COPY:
+                dest, source = fields[1:]
+                self.add_copy(source, dest)
+                continue
+
+            if record_type == self.REQUIRED_EXISTS:
+                _, path = fields
+                self.add_required_exists(path)
+                continue
+
+            if record_type == self.OPTIONAL_EXISTS:
+                _, path = fields
+                self.add_optional_exists(path)
+                continue
+
+            if record_type == self.PATTERN_LINK:
+                _, base, pattern, dest = fields[1:]
+                self.add_pattern_link(base, pattern, dest)
+                continue
+
+            if record_type == self.PATTERN_COPY:
+                _, base, pattern, dest = fields[1:]
+                self.add_pattern_copy(base, pattern, dest)
+                continue
+
+            if record_type == self.PREPROCESS:
+                dest, source, deps, marker, defines, warnings = fields[1:]
+
+                self.add_preprocess(
+                    source,
+                    dest,
+                    deps,
+                    marker,
+                    self._decode_field_entry(defines),
+                    silence_missing_directive_warnings=bool(int(warnings)),
+                )
+                continue
+
+            if record_type == self.CONTENT:
+                dest, content = fields[1:]
+
+                self.add_content(
+                    six.ensure_text(self._decode_field_entry(content)), dest
+                )
+                continue
+
+            # Don't fail for non-actionable items, allowing
+            # forward-compatibility with those we will add in the future.
+            if record_type >= 0:
+                raise UnreadableInstallManifest("Unknown record type: %d" % record_type)
+
+    def __len__(self):
+        return len(self._dests)
+
+    def __contains__(self, item):
+        return item in self._dests
+
+    def __eq__(self, other):
+        return isinstance(other, InstallManifest) and self._dests == other._dests
+
+    def __neq__(self, other):
+        return not self.__eq__(other)
+
+    def __ior__(self, other):
+        if not isinstance(other, InstallManifest):
+            raise ValueError("Can only | with another instance of InstallManifest.")
+
+        self.add_entries_from(other)
+
+        return self
+
+    def _encode_field_entry(self, data):
+        """Converts an object into a format that can be stored in the manifest file.
+
+        Complex data types, such as ``dict``, need to be converted into a text
+        representation before they can be written to a file.
+        """
+        return json.dumps(data, sort_keys=True)
+
+    def _decode_field_entry(self, data):
+        """Restores an object from a format that can be stored in the manifest file.
+
+        Complex data types, such as ``dict``, need to be converted into a text
+        representation before they can be written to a file.
+        """
+        return json.loads(data)
+
+    def write(self, path=None, fileobj=None, expand_pattern=False):
+        """Serialize this manifest to a file or file object.
+
+        If path is specified, that file will be written to. If fileobj is specified,
+        the serialized content will be written to that file object.
+
+        It is an error if both are specified.
+        """
+        with _auto_fileobj(path, fileobj, "wt") as fh:
+            fh.write("%d\n" % self.CURRENT_VERSION)
+
+            for dest in sorted(self._dests):
+                entry = self._dests[dest]
+
+                if expand_pattern and entry[0] in (
+                    self.PATTERN_LINK,
+                    self.PATTERN_COPY,
+                ):
+                    type, base, pattern, dest = entry
+                    type = self.LINK if type == self.PATTERN_LINK else self.COPY
+                    finder = FileFinder(base)
+                    paths = [f[0] for f in finder.find(pattern)]
+                    for path in paths:
+                        source = mozpath.join(base, path)
+                        parts = ["%d" % type, mozpath.join(dest, path), source]
+                        fh.write(
+                            "%s\n"
+                            % self.FIELD_SEPARATOR.join(
+                                six.ensure_text(p) for p in parts
+                            )
+                        )
+                else:
+                    parts = ["%d" % entry[0], dest]
+                    parts.extend(entry[1:])
+                    fh.write(
+                        "%s\n"
+                        % self.FIELD_SEPARATOR.join(six.ensure_text(p) for p in parts)
+                    )
+
+    def add_link(self, source, dest):
+        """Add a link to this manifest.
+
+        dest will be either a symlink or hardlink to source.
+        """
+        self._add_entry(dest, (self.LINK, source))
+
+    def add_copy(self, source, dest):
+        """Add a copy to this manifest.
+
+        source will be copied to dest.
+        """
+        self._add_entry(dest, (self.COPY, source))
+
+    def add_required_exists(self, dest):
+        """Record that a destination file must exist.
+
+        This effectively prevents the listed file from being deleted.
+        """
+        self._add_entry(dest, (self.REQUIRED_EXISTS,))
+
+    def add_optional_exists(self, dest):
+        """Record that a destination file may exist.
+
+        This effectively prevents the listed file from being deleted. Unlike a
+        "required exists" file, files of this type do not raise errors if the
+        destination file does not exist.
+        """
+        self._add_entry(dest, (self.OPTIONAL_EXISTS,))
+
+    def add_pattern_link(self, base, pattern, dest):
+        """Add a pattern match that results in links being created.
+
+        A ``FileFinder`` will be created with its base set to ``base``
+        and ``FileFinder.find()`` will be called with ``pattern`` to discover
+        source files. Each source file will be either symlinked or hardlinked
+        under ``dest``.
+
+        Filenames under ``dest`` are constructed by taking the path fragment
+        after ``base`` and concatenating it with ``dest``. e.g.
+
+           <base>/foo/bar.h -> <dest>/foo/bar.h
+        """
+        self._add_entry(
+            mozpath.join(dest, pattern), (self.PATTERN_LINK, base, pattern, dest)
+        )
+
+    def add_pattern_copy(self, base, pattern, dest):
+        """Add a pattern match that results in copies.
+
+        See ``add_pattern_link()`` for usage.
+        """
+        self._add_entry(
+            mozpath.join(dest, pattern), (self.PATTERN_COPY, base, pattern, dest)
+        )
+
+    def add_preprocess(
+        self,
+        source,
+        dest,
+        deps,
+        marker="#",
+        defines={},
+        silence_missing_directive_warnings=False,
+    ):
+        """Add a preprocessed file to this manifest.
+
+        ``source`` will be passed through preprocessor.py, and the output will be
+        written to ``dest``.
+        """
+        self._add_entry(
+            dest,
+            (
+                self.PREPROCESS,
+                source,
+                deps,
+                marker,
+                self._encode_field_entry(defines),
+                "1" if silence_missing_directive_warnings else "0",
+            ),
+        )
+
+    def add_content(self, content, dest):
+        """Add a file with the given content."""
+        self._add_entry(
+            dest,
+            (
+                self.CONTENT,
+                self._encode_field_entry(content),
+            ),
+        )
+
+    def _add_entry(self, dest, entry):
+        if dest in self._dests:
+            raise ValueError("Item already in manifest: %s" % dest)
+
+        self._dests[dest] = entry
+
+    def add_entries_from(self, other, base=""):
+        """
+        Copy data from another mozpack.copier.InstallManifest
+        instance, adding an optional base prefix to the destination.
+
+        This allows to merge two manifests into a single manifest, or
+        two take the tagged union of two manifests.
+        """
+        # We must copy source files to ourselves so extra dependencies from
+        # the preprocessor are taken into account. Ideally, we would track
+        # which source file each entry came from. However, this is more
+        # complicated and not yet implemented. The current implementation
+        # will result in over invalidation, possibly leading to performance
+        # loss.
+        self._source_files |= other._source_files
+
+        for dest in sorted(other._dests):
+            new_dest = mozpath.join(base, dest) if base else dest
+            entry = other._dests[dest]
+            if entry[0] in (self.PATTERN_LINK, self.PATTERN_COPY):
+                entry_type, entry_base, entry_pattern, entry_dest = entry
+                new_entry_dest = mozpath.join(base, entry_dest) if base else entry_dest
+                new_entry = (entry_type, entry_base, entry_pattern, new_entry_dest)
+            else:
+                new_entry = tuple(entry)
+
+            self._add_entry(new_dest, new_entry)
+
+    def populate_registry(self, registry, defines_override={}, link_policy="symlink"):
+        """Populate a mozpack.copier.FileRegistry instance with data from us.
+
+        The caller supplied a FileRegistry instance (or at least something that
+        conforms to its interface) and that instance is populated with data
+        from this manifest.
+
+        Defines can be given to override the ones in the manifest for
+        preprocessing.
+
+        The caller can set a link policy. This determines whether symlinks,
+        hardlinks, or copies are used for LINK and PATTERN_LINK.
+        """
+        assert link_policy in ("symlink", "hardlink", "copy")
+        for dest in sorted(self._dests):
+            entry = self._dests[dest]
+            install_type = entry[0]
+
+            if install_type == self.LINK:
+                if link_policy == "symlink":
+                    cls = AbsoluteSymlinkFile
+                elif link_policy == "hardlink":
+                    cls = HardlinkFile
+                else:
+                    cls = File
+                registry.add(dest, cls(entry[1]))
+                continue
+
+            if install_type == self.COPY:
+                registry.add(dest, File(entry[1]))
+                continue
+
+            if install_type == self.REQUIRED_EXISTS:
+                registry.add(dest, ExistingFile(required=True))
+                continue
+
+            if install_type == self.OPTIONAL_EXISTS:
+                registry.add(dest, ExistingFile(required=False))
+                continue
+
+            if install_type in (self.PATTERN_LINK, self.PATTERN_COPY):
+                _, base, pattern, dest = entry
+                finder = FileFinder(base)
+                paths = [f[0] for f in finder.find(pattern)]
+
+                if install_type == self.PATTERN_LINK:
+                    if link_policy == "symlink":
+                        cls = AbsoluteSymlinkFile
+                    elif link_policy == "hardlink":
+                        cls = HardlinkFile
+                    else:
+                        cls = File
+                else:
+                    cls = File
+
+                for path in paths:
+                    source = mozpath.join(base, path)
+                    registry.add(mozpath.join(dest, path), cls(source))
+
+                continue
+
+            if install_type == self.PREPROCESS:
+                defines = self._decode_field_entry(entry[4])
+                if defines_override:
+                    defines.update(defines_override)
+                registry.add(
+                    dest,
+                    PreprocessedFile(
+                        entry[1],
+                        depfile_path=entry[2],
+                        marker=entry[3],
+                        defines=defines,
+                        extra_depends=self._source_files,
+                        silence_missing_directive_warnings=bool(int(entry[5])),
+                    ),
+                )
+
+                continue
+
+            if install_type == self.CONTENT:
+                # GeneratedFile expect the buffer interface, which the unicode
+                # type doesn't have, so encode to a str.
+                content = self._decode_field_entry(entry[1]).encode("utf-8")
+                registry.add(dest, GeneratedFile(content))
+                continue
+
+            raise Exception(
+                "Unknown install type defined in manifest: %d" % install_type
+            )