diff options
Diffstat (limited to 'python/mozbuild/mozpack/packager/__init__.py')
| -rw-r--r-- | python/mozbuild/mozpack/packager/__init__.py | 445 |
1 files changed, 445 insertions, 0 deletions
diff --git a/python/mozbuild/mozpack/packager/__init__.py b/python/mozbuild/mozpack/packager/__init__.py new file mode 100644 index 0000000000..83b12e4696 --- /dev/null +++ b/python/mozbuild/mozpack/packager/__init__.py @@ -0,0 +1,445 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import codecs +import json +import os +import re +from collections import deque + +import six + +import mozpack.path as mozpath +from mozbuild.preprocessor import Preprocessor +from mozpack.chrome.manifest import ( + Manifest, + ManifestBinaryComponent, + ManifestChrome, + ManifestInterfaces, + is_manifest, + parse_manifest, +) +from mozpack.errors import errors + + +class Component(object): + """ + Class that represents a component in a package manifest. + """ + + def __init__(self, name, destdir=""): + if name.find(" ") > 0: + errors.fatal('Malformed manifest: space in component name "%s"' % name) + self._name = name + self._destdir = destdir + + def __repr__(self): + s = self.name + if self.destdir: + s += ' destdir="%s"' % self.destdir + return s + + @property + def name(self): + return self._name + + @property + def destdir(self): + return self._destdir + + @staticmethod + def _triples(lst): + """ + Split [1, 2, 3, 4, 5, 6, 7] into [(1, 2, 3), (4, 5, 6)]. + """ + return zip(*[iter(lst)] * 3) + + KEY_VALUE_RE = re.compile( + r""" + \s* # optional whitespace. + ([a-zA-Z0-9_]+) # key. + \s*=\s* # optional space around =. + "([^"]*)" # value without surrounding quotes. + (?:\s+|$) + """, + re.VERBOSE, + ) + + @staticmethod + def _split_options(string): + """ + Split 'key1="value1" key2="value2"' into + {'key1':'value1', 'key2':'value2'}. + + Returned keys and values are all strings. + + Throws ValueError if the input is malformed. + """ + options = {} + splits = Component.KEY_VALUE_RE.split(string) + if len(splits) % 3 != 1: + # This should never happen -- we expect to always split + # into ['', ('key', 'val', '')*]. + raise ValueError("Bad input") + if splits[0]: + raise ValueError("Unrecognized input " + splits[0]) + for key, val, no_match in Component._triples(splits[1:]): + if no_match: + raise ValueError("Unrecognized input " + no_match) + options[key] = val + return options + + @staticmethod + def _split_component_and_options(string): + """ + Split 'name key1="value1" key2="value2"' into + ('name', {'key1':'value1', 'key2':'value2'}). + + Returned name, keys and values are all strings. + + Raises ValueError if the input is malformed. + """ + splits = string.strip().split(None, 1) + if not splits: + raise ValueError("No component found") + component = splits[0].strip() + if not component: + raise ValueError("No component found") + if not re.match("[a-zA-Z0-9_\-]+$", component): + raise ValueError("Bad component name " + component) + options = Component._split_options(splits[1]) if len(splits) > 1 else {} + return component, options + + @staticmethod + def from_string(string): + """ + Create a component from a string. + """ + try: + name, options = Component._split_component_and_options(string) + except ValueError as e: + errors.fatal("Malformed manifest: %s" % e) + return + destdir = options.pop("destdir", "") + if options: + errors.fatal( + "Malformed manifest: options %s not recognized" % options.keys() + ) + return Component(name, destdir=destdir) + + +class PackageManifestParser(object): + """ + Class for parsing of a package manifest, after preprocessing. + + A package manifest is a list of file paths, with some syntaxic sugar: + [] designates a toplevel component. Example: [xpcom] + - in front of a file specifies it to be removed + * wildcard support + ** expands to all files and zero or more directories + ; file comment + + The parser takes input from the preprocessor line by line, and pushes + parsed information to a sink object. + + The add and remove methods of the sink object are called with the + current Component instance and a path. + """ + + def __init__(self, sink): + """ + Initialize the package manifest parser with the given sink. + """ + self._component = Component("") + self._sink = sink + + def handle_line(self, str): + """ + Handle a line of input and push the parsed information to the sink + object. + """ + # Remove comments. + str = str.strip() + if not str or str.startswith(";"): + return + if str.startswith("[") and str.endswith("]"): + self._component = Component.from_string(str[1:-1]) + elif str.startswith("-"): + str = str[1:] + self._sink.remove(self._component, str) + elif "," in str: + errors.fatal("Incompatible syntax") + else: + self._sink.add(self._component, str) + + +class PreprocessorOutputWrapper(object): + """ + File-like helper to handle the preprocessor output and send it to a parser. + The parser's handle_line method is called in the relevant errors.context. + """ + + def __init__(self, preprocessor, parser): + self._parser = parser + self._pp = preprocessor + + def write(self, str): + with errors.context(self._pp.context["FILE"], self._pp.context["LINE"]): + self._parser.handle_line(str) + + +def preprocess(input, parser, defines={}): + """ + Preprocess the file-like input with the given defines, and send the + preprocessed output line by line to the given parser. + """ + pp = Preprocessor() + pp.context.update(defines) + pp.do_filter("substitution") + pp.out = PreprocessorOutputWrapper(pp, parser) + pp.do_include(input) + + +def preprocess_manifest(sink, manifest, defines={}): + """ + Preprocess the given file-like manifest with the given defines, and push + the parsed information to a sink. See PackageManifestParser documentation + for more details on the sink. + """ + preprocess(manifest, PackageManifestParser(sink), defines) + + +class CallDeque(deque): + """ + Queue of function calls to make. + """ + + def append(self, function, *args): + deque.append(self, (errors.get_context(), function, args)) + + def execute(self): + while True: + try: + context, function, args = self.popleft() + except IndexError: + return + if context: + with errors.context(context[0], context[1]): + function(*args) + else: + function(*args) + + +class SimplePackager(object): + """ + Helper used to translate and buffer instructions from the + SimpleManifestSink to a formatter. Formatters expect some information to be + given first that the simple manifest contents can't guarantee before the + end of the input. + """ + + def __init__(self, formatter): + self.formatter = formatter + # Queue for formatter.add_interfaces()/add_manifest() calls. + self._queue = CallDeque() + # Queue for formatter.add_manifest() calls for ManifestChrome. + self._chrome_queue = CallDeque() + # Queue for formatter.add() calls. + self._file_queue = CallDeque() + # All paths containing addons. (key is path, value is whether it + # should be packed or unpacked) + self._addons = {} + # All manifest paths imported. + self._manifests = set() + # All manifest paths included from some other manifest. + self._included_manifests = {} + self._closed = False + + # Parsing RDF is complex, and would require an external library to do + # properly. Just go with some hackish but probably sufficient regexp + UNPACK_ADDON_RE = re.compile( + r"""(?: + <em:unpack>true</em:unpack> + |em:unpack=(?P<quote>["']?)true(?P=quote) + )""", + re.VERBOSE, + ) + + def add(self, path, file): + """ + Add the given BaseFile instance with the given path. + """ + assert not self._closed + if is_manifest(path): + self._add_manifest_file(path, file) + elif path.endswith(".xpt"): + self._queue.append(self.formatter.add_interfaces, path, file) + else: + self._file_queue.append(self.formatter.add, path, file) + if mozpath.basename(path) == "install.rdf": + addon = True + install_rdf = six.ensure_text(file.open().read()) + if self.UNPACK_ADDON_RE.search(install_rdf): + addon = "unpacked" + self._add_addon(mozpath.dirname(path), addon) + elif mozpath.basename(path) == "manifest.json": + manifest = six.ensure_text(file.open().read()) + try: + parsed = json.loads(manifest) + except ValueError: + pass + if isinstance(parsed, dict) and "manifest_version" in parsed: + self._add_addon(mozpath.dirname(path), True) + + def _add_addon(self, path, addon_type): + """ + Add the given BaseFile to the collection of addons if a parent + directory is not already in the collection. + """ + if mozpath.basedir(path, self._addons) is not None: + return + + for dir in self._addons: + if mozpath.basedir(dir, [path]) is not None: + del self._addons[dir] + break + + self._addons[path] = addon_type + + def _add_manifest_file(self, path, file): + """ + Add the given BaseFile with manifest file contents with the given path. + """ + self._manifests.add(path) + base = "" + if hasattr(file, "path"): + # Find the directory the given path is relative to. + b = mozpath.normsep(file.path) + if b.endswith("/" + path) or b == path: + base = os.path.normpath(b[: -len(path)]) + for e in parse_manifest(base, path, codecs.getreader("utf-8")(file.open())): + # ManifestResources need to be given after ManifestChrome, so just + # put all ManifestChrome in a separate queue to make them first. + if isinstance(e, ManifestChrome): + # e.move(e.base) just returns a clone of the entry. + self._chrome_queue.append(self.formatter.add_manifest, e.move(e.base)) + elif not isinstance(e, (Manifest, ManifestInterfaces)): + self._queue.append(self.formatter.add_manifest, e.move(e.base)) + # If a binary component is added to an addon, prevent the addon + # from being packed. + if isinstance(e, ManifestBinaryComponent): + addon = mozpath.basedir(e.base, self._addons) + if addon: + self._addons[addon] = "unpacked" + if isinstance(e, Manifest): + if e.flags: + errors.fatal("Flags are not supported on " + '"manifest" entries') + self._included_manifests[e.path] = path + + def get_bases(self, addons=True): + """ + Return all paths under which root manifests have been found. Root + manifests are manifests that are included in no other manifest. + `addons` indicates whether to include addon bases as well. + """ + all_bases = set( + mozpath.dirname(m) for m in self._manifests - set(self._included_manifests) + ) + if not addons: + all_bases -= set(self._addons) + else: + # If for some reason some detected addon doesn't have a + # non-included manifest. + all_bases |= set(self._addons) + return all_bases + + def close(self): + """ + Push all instructions to the formatter. + """ + self._closed = True + + bases = self.get_bases() + broken_bases = sorted( + m + for m, includer in six.iteritems(self._included_manifests) + if mozpath.basedir(m, bases) != mozpath.basedir(includer, bases) + ) + for m in broken_bases: + errors.fatal( + '"%s" is included from "%s", which is outside "%s"' + % (m, self._included_manifests[m], mozpath.basedir(m, bases)) + ) + for base in sorted(bases): + self.formatter.add_base(base, self._addons.get(base, False)) + self._chrome_queue.execute() + self._queue.execute() + self._file_queue.execute() + + +class SimpleManifestSink(object): + """ + Parser sink for "simple" package manifests. Simple package manifests use + the format described in the PackageManifestParser documentation, but don't + support file removals, and require manifests, interfaces and chrome data to + be explicitely listed. + Entries starting with bin/ are searched under bin/ in the FileFinder, but + are packaged without the bin/ prefix. + """ + + def __init__(self, finder, formatter): + """ + Initialize the SimpleManifestSink. The given FileFinder is used to + get files matching the patterns given in the manifest. The given + formatter does the packaging job. + """ + self._finder = finder + self.packager = SimplePackager(formatter) + self._closed = False + self._manifests = set() + + @staticmethod + def normalize_path(path): + """ + Remove any bin/ prefix. + """ + if mozpath.basedir(path, ["bin"]) == "bin": + return mozpath.relpath(path, "bin") + return path + + def add(self, component, pattern): + """ + Add files with the given pattern in the given component. + """ + assert not self._closed + added = False + for p, f in self._finder.find(pattern): + added = True + if is_manifest(p): + self._manifests.add(p) + dest = mozpath.join(component.destdir, SimpleManifestSink.normalize_path(p)) + self.packager.add(dest, f) + if not added: + errors.error("Missing file(s): %s" % pattern) + + def remove(self, component, pattern): + """ + Remove files with the given pattern in the given component. + """ + assert not self._closed + errors.fatal("Removal is unsupported") + + def close(self, auto_root_manifest=True): + """ + Add possibly missing bits and push all instructions to the formatter. + """ + if auto_root_manifest: + # Simple package manifests don't contain the root manifests, so + # find and add them. + paths = [mozpath.dirname(m) for m in self._manifests] + path = mozpath.dirname(mozpath.commonprefix(paths)) + for p, f in self._finder.find(mozpath.join(path, "chrome.manifest")): + if p not in self._manifests: + self.packager.add(SimpleManifestSink.normalize_path(p), f) + self.packager.close() |