diff options
Diffstat (limited to 'python/mozbuild/mozpack/packager')
| -rw-r--r-- | python/mozbuild/mozpack/packager/__init__.py | 445 | ||||
| -rw-r--r-- | python/mozbuild/mozpack/packager/formats.py | 354 | ||||
| -rw-r--r-- | python/mozbuild/mozpack/packager/l10n.py | 304 | ||||
| -rw-r--r-- | python/mozbuild/mozpack/packager/unpack.py | 200 |
4 files changed, 1303 insertions, 0 deletions
diff --git a/python/mozbuild/mozpack/packager/__init__.py b/python/mozbuild/mozpack/packager/__init__.py new file mode 100644 index 0000000000..83b12e4696 --- /dev/null +++ b/python/mozbuild/mozpack/packager/__init__.py @@ -0,0 +1,445 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import codecs +import json +import os +import re +from collections import deque + +import six + +import mozpack.path as mozpath +from mozbuild.preprocessor import Preprocessor +from mozpack.chrome.manifest import ( + Manifest, + ManifestBinaryComponent, + ManifestChrome, + ManifestInterfaces, + is_manifest, + parse_manifest, +) +from mozpack.errors import errors + + +class Component(object): + """ + Class that represents a component in a package manifest. + """ + + def __init__(self, name, destdir=""): + if name.find(" ") > 0: + errors.fatal('Malformed manifest: space in component name "%s"' % name) + self._name = name + self._destdir = destdir + + def __repr__(self): + s = self.name + if self.destdir: + s += ' destdir="%s"' % self.destdir + return s + + @property + def name(self): + return self._name + + @property + def destdir(self): + return self._destdir + + @staticmethod + def _triples(lst): + """ + Split [1, 2, 3, 4, 5, 6, 7] into [(1, 2, 3), (4, 5, 6)]. + """ + return zip(*[iter(lst)] * 3) + + KEY_VALUE_RE = re.compile( + r""" + \s* # optional whitespace. + ([a-zA-Z0-9_]+) # key. + \s*=\s* # optional space around =. + "([^"]*)" # value without surrounding quotes. + (?:\s+|$) + """, + re.VERBOSE, + ) + + @staticmethod + def _split_options(string): + """ + Split 'key1="value1" key2="value2"' into + {'key1':'value1', 'key2':'value2'}. + + Returned keys and values are all strings. + + Throws ValueError if the input is malformed. + """ + options = {} + splits = Component.KEY_VALUE_RE.split(string) + if len(splits) % 3 != 1: + # This should never happen -- we expect to always split + # into ['', ('key', 'val', '')*]. + raise ValueError("Bad input") + if splits[0]: + raise ValueError("Unrecognized input " + splits[0]) + for key, val, no_match in Component._triples(splits[1:]): + if no_match: + raise ValueError("Unrecognized input " + no_match) + options[key] = val + return options + + @staticmethod + def _split_component_and_options(string): + """ + Split 'name key1="value1" key2="value2"' into + ('name', {'key1':'value1', 'key2':'value2'}). + + Returned name, keys and values are all strings. + + Raises ValueError if the input is malformed. + """ + splits = string.strip().split(None, 1) + if not splits: + raise ValueError("No component found") + component = splits[0].strip() + if not component: + raise ValueError("No component found") + if not re.match("[a-zA-Z0-9_\-]+$", component): + raise ValueError("Bad component name " + component) + options = Component._split_options(splits[1]) if len(splits) > 1 else {} + return component, options + + @staticmethod + def from_string(string): + """ + Create a component from a string. + """ + try: + name, options = Component._split_component_and_options(string) + except ValueError as e: + errors.fatal("Malformed manifest: %s" % e) + return + destdir = options.pop("destdir", "") + if options: + errors.fatal( + "Malformed manifest: options %s not recognized" % options.keys() + ) + return Component(name, destdir=destdir) + + +class PackageManifestParser(object): + """ + Class for parsing of a package manifest, after preprocessing. + + A package manifest is a list of file paths, with some syntaxic sugar: + [] designates a toplevel component. Example: [xpcom] + - in front of a file specifies it to be removed + * wildcard support + ** expands to all files and zero or more directories + ; file comment + + The parser takes input from the preprocessor line by line, and pushes + parsed information to a sink object. + + The add and remove methods of the sink object are called with the + current Component instance and a path. + """ + + def __init__(self, sink): + """ + Initialize the package manifest parser with the given sink. + """ + self._component = Component("") + self._sink = sink + + def handle_line(self, str): + """ + Handle a line of input and push the parsed information to the sink + object. + """ + # Remove comments. + str = str.strip() + if not str or str.startswith(";"): + return + if str.startswith("[") and str.endswith("]"): + self._component = Component.from_string(str[1:-1]) + elif str.startswith("-"): + str = str[1:] + self._sink.remove(self._component, str) + elif "," in str: + errors.fatal("Incompatible syntax") + else: + self._sink.add(self._component, str) + + +class PreprocessorOutputWrapper(object): + """ + File-like helper to handle the preprocessor output and send it to a parser. + The parser's handle_line method is called in the relevant errors.context. + """ + + def __init__(self, preprocessor, parser): + self._parser = parser + self._pp = preprocessor + + def write(self, str): + with errors.context(self._pp.context["FILE"], self._pp.context["LINE"]): + self._parser.handle_line(str) + + +def preprocess(input, parser, defines={}): + """ + Preprocess the file-like input with the given defines, and send the + preprocessed output line by line to the given parser. + """ + pp = Preprocessor() + pp.context.update(defines) + pp.do_filter("substitution") + pp.out = PreprocessorOutputWrapper(pp, parser) + pp.do_include(input) + + +def preprocess_manifest(sink, manifest, defines={}): + """ + Preprocess the given file-like manifest with the given defines, and push + the parsed information to a sink. See PackageManifestParser documentation + for more details on the sink. + """ + preprocess(manifest, PackageManifestParser(sink), defines) + + +class CallDeque(deque): + """ + Queue of function calls to make. + """ + + def append(self, function, *args): + deque.append(self, (errors.get_context(), function, args)) + + def execute(self): + while True: + try: + context, function, args = self.popleft() + except IndexError: + return + if context: + with errors.context(context[0], context[1]): + function(*args) + else: + function(*args) + + +class SimplePackager(object): + """ + Helper used to translate and buffer instructions from the + SimpleManifestSink to a formatter. Formatters expect some information to be + given first that the simple manifest contents can't guarantee before the + end of the input. + """ + + def __init__(self, formatter): + self.formatter = formatter + # Queue for formatter.add_interfaces()/add_manifest() calls. + self._queue = CallDeque() + # Queue for formatter.add_manifest() calls for ManifestChrome. + self._chrome_queue = CallDeque() + # Queue for formatter.add() calls. + self._file_queue = CallDeque() + # All paths containing addons. (key is path, value is whether it + # should be packed or unpacked) + self._addons = {} + # All manifest paths imported. + self._manifests = set() + # All manifest paths included from some other manifest. + self._included_manifests = {} + self._closed = False + + # Parsing RDF is complex, and would require an external library to do + # properly. Just go with some hackish but probably sufficient regexp + UNPACK_ADDON_RE = re.compile( + r"""(?: + <em:unpack>true</em:unpack> + |em:unpack=(?P<quote>["']?)true(?P=quote) + )""", + re.VERBOSE, + ) + + def add(self, path, file): + """ + Add the given BaseFile instance with the given path. + """ + assert not self._closed + if is_manifest(path): + self._add_manifest_file(path, file) + elif path.endswith(".xpt"): + self._queue.append(self.formatter.add_interfaces, path, file) + else: + self._file_queue.append(self.formatter.add, path, file) + if mozpath.basename(path) == "install.rdf": + addon = True + install_rdf = six.ensure_text(file.open().read()) + if self.UNPACK_ADDON_RE.search(install_rdf): + addon = "unpacked" + self._add_addon(mozpath.dirname(path), addon) + elif mozpath.basename(path) == "manifest.json": + manifest = six.ensure_text(file.open().read()) + try: + parsed = json.loads(manifest) + except ValueError: + pass + if isinstance(parsed, dict) and "manifest_version" in parsed: + self._add_addon(mozpath.dirname(path), True) + + def _add_addon(self, path, addon_type): + """ + Add the given BaseFile to the collection of addons if a parent + directory is not already in the collection. + """ + if mozpath.basedir(path, self._addons) is not None: + return + + for dir in self._addons: + if mozpath.basedir(dir, [path]) is not None: + del self._addons[dir] + break + + self._addons[path] = addon_type + + def _add_manifest_file(self, path, file): + """ + Add the given BaseFile with manifest file contents with the given path. + """ + self._manifests.add(path) + base = "" + if hasattr(file, "path"): + # Find the directory the given path is relative to. + b = mozpath.normsep(file.path) + if b.endswith("/" + path) or b == path: + base = os.path.normpath(b[: -len(path)]) + for e in parse_manifest(base, path, codecs.getreader("utf-8")(file.open())): + # ManifestResources need to be given after ManifestChrome, so just + # put all ManifestChrome in a separate queue to make them first. + if isinstance(e, ManifestChrome): + # e.move(e.base) just returns a clone of the entry. + self._chrome_queue.append(self.formatter.add_manifest, e.move(e.base)) + elif not isinstance(e, (Manifest, ManifestInterfaces)): + self._queue.append(self.formatter.add_manifest, e.move(e.base)) + # If a binary component is added to an addon, prevent the addon + # from being packed. + if isinstance(e, ManifestBinaryComponent): + addon = mozpath.basedir(e.base, self._addons) + if addon: + self._addons[addon] = "unpacked" + if isinstance(e, Manifest): + if e.flags: + errors.fatal("Flags are not supported on " + '"manifest" entries') + self._included_manifests[e.path] = path + + def get_bases(self, addons=True): + """ + Return all paths under which root manifests have been found. Root + manifests are manifests that are included in no other manifest. + `addons` indicates whether to include addon bases as well. + """ + all_bases = set( + mozpath.dirname(m) for m in self._manifests - set(self._included_manifests) + ) + if not addons: + all_bases -= set(self._addons) + else: + # If for some reason some detected addon doesn't have a + # non-included manifest. + all_bases |= set(self._addons) + return all_bases + + def close(self): + """ + Push all instructions to the formatter. + """ + self._closed = True + + bases = self.get_bases() + broken_bases = sorted( + m + for m, includer in six.iteritems(self._included_manifests) + if mozpath.basedir(m, bases) != mozpath.basedir(includer, bases) + ) + for m in broken_bases: + errors.fatal( + '"%s" is included from "%s", which is outside "%s"' + % (m, self._included_manifests[m], mozpath.basedir(m, bases)) + ) + for base in sorted(bases): + self.formatter.add_base(base, self._addons.get(base, False)) + self._chrome_queue.execute() + self._queue.execute() + self._file_queue.execute() + + +class SimpleManifestSink(object): + """ + Parser sink for "simple" package manifests. Simple package manifests use + the format described in the PackageManifestParser documentation, but don't + support file removals, and require manifests, interfaces and chrome data to + be explicitely listed. + Entries starting with bin/ are searched under bin/ in the FileFinder, but + are packaged without the bin/ prefix. + """ + + def __init__(self, finder, formatter): + """ + Initialize the SimpleManifestSink. The given FileFinder is used to + get files matching the patterns given in the manifest. The given + formatter does the packaging job. + """ + self._finder = finder + self.packager = SimplePackager(formatter) + self._closed = False + self._manifests = set() + + @staticmethod + def normalize_path(path): + """ + Remove any bin/ prefix. + """ + if mozpath.basedir(path, ["bin"]) == "bin": + return mozpath.relpath(path, "bin") + return path + + def add(self, component, pattern): + """ + Add files with the given pattern in the given component. + """ + assert not self._closed + added = False + for p, f in self._finder.find(pattern): + added = True + if is_manifest(p): + self._manifests.add(p) + dest = mozpath.join(component.destdir, SimpleManifestSink.normalize_path(p)) + self.packager.add(dest, f) + if not added: + errors.error("Missing file(s): %s" % pattern) + + def remove(self, component, pattern): + """ + Remove files with the given pattern in the given component. + """ + assert not self._closed + errors.fatal("Removal is unsupported") + + def close(self, auto_root_manifest=True): + """ + Add possibly missing bits and push all instructions to the formatter. + """ + if auto_root_manifest: + # Simple package manifests don't contain the root manifests, so + # find and add them. + paths = [mozpath.dirname(m) for m in self._manifests] + path = mozpath.dirname(mozpath.commonprefix(paths)) + for p, f in self._finder.find(mozpath.join(path, "chrome.manifest")): + if p not in self._manifests: + self.packager.add(SimpleManifestSink.normalize_path(p), f) + self.packager.close() diff --git a/python/mozbuild/mozpack/packager/formats.py b/python/mozbuild/mozpack/packager/formats.py new file mode 100644 index 0000000000..95a6dee2f6 --- /dev/null +++ b/python/mozbuild/mozpack/packager/formats.py @@ -0,0 +1,354 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from six.moves.urllib.parse import urlparse + +import mozpack.path as mozpath +from mozpack.chrome.manifest import ( + Manifest, + ManifestBinaryComponent, + ManifestChrome, + ManifestInterfaces, + ManifestMultiContent, + ManifestResource, +) +from mozpack.copier import FileRegistry, FileRegistrySubtree, Jarrer +from mozpack.errors import errors +from mozpack.files import ManifestFile + +""" +Formatters are classes receiving packaging instructions and creating the +appropriate package layout. + +There are three distinct formatters, each handling one of the different chrome +formats: + - flat: essentially, copies files from the source with the same file system + layout. Manifests entries are grouped in a single manifest per directory, + as well as XPT interfaces. + - jar: chrome content is packaged in jar files. + - omni: chrome content, modules, non-binary components, and many other + elements are packaged in an omnijar file for each base directory. + +The base interface provides the following methods: + - add_base(path [, addon]) + Register a base directory for an application or GRE, or an addon. + Base directories usually contain a root manifest (manifests not + included in any other manifest) named chrome.manifest. + The optional addon argument tells whether the base directory + is that of a packed addon (True), unpacked addon ('unpacked') or + otherwise (False). + The method may only be called in sorted order of `path` (alphanumeric + order, parents before children). + - add(path, content) + Add the given content (BaseFile instance) at the given virtual path + - add_interfaces(path, content) + Add the given content (BaseFile instance) as an interface. Equivalent + to add(path, content) with the right add_manifest(). + - add_manifest(entry) + Add a ManifestEntry. + - contains(path) + Returns whether the given virtual path is known of the formatter. + +The virtual paths mentioned above are paths as they would be with a flat +chrome. + +Formatters all take a FileCopier instance they will fill with the packaged +data. +""" + + +class PiecemealFormatter(object): + """ + Generic formatter that dispatches across different sub-formatters + according to paths. + """ + + def __init__(self, copier): + assert isinstance(copier, (FileRegistry, FileRegistrySubtree)) + self.copier = copier + self._sub_formatter = {} + self._frozen_bases = False + + def add_base(self, base, addon=False): + # Only allow to add a base directory before calls to _get_base() + assert not self._frozen_bases + assert base not in self._sub_formatter + assert all(base > b for b in self._sub_formatter) + self._add_base(base, addon) + + def _get_base(self, path): + """ + Return the deepest base directory containing the given path. + """ + self._frozen_bases = True + base = mozpath.basedir(path, self._sub_formatter.keys()) + relpath = mozpath.relpath(path, base) if base else path + return base, relpath + + def add(self, path, content): + base, relpath = self._get_base(path) + if base is None: + return self.copier.add(relpath, content) + return self._sub_formatter[base].add(relpath, content) + + def add_manifest(self, entry): + base, relpath = self._get_base(entry.base) + assert base is not None + return self._sub_formatter[base].add_manifest(entry.move(relpath)) + + def add_interfaces(self, path, content): + base, relpath = self._get_base(path) + assert base is not None + return self._sub_formatter[base].add_interfaces(relpath, content) + + def contains(self, path): + assert "*" not in path + base, relpath = self._get_base(path) + if base is None: + return self.copier.contains(relpath) + return self._sub_formatter[base].contains(relpath) + + +class FlatFormatter(PiecemealFormatter): + """ + Formatter for the flat package format. + """ + + def _add_base(self, base, addon=False): + self._sub_formatter[base] = FlatSubFormatter( + FileRegistrySubtree(base, self.copier) + ) + + +class FlatSubFormatter(object): + """ + Sub-formatter for the flat package format. + """ + + def __init__(self, copier): + assert isinstance(copier, (FileRegistry, FileRegistrySubtree)) + self.copier = copier + self._chrome_db = {} + + def add(self, path, content): + self.copier.add(path, content) + + def add_manifest(self, entry): + # Store manifest entries in a single manifest per directory, named + # after their parent directory, except for root manifests, all named + # chrome.manifest. + if entry.base: + name = mozpath.basename(entry.base) + else: + name = "chrome" + path = mozpath.normpath(mozpath.join(entry.base, "%s.manifest" % name)) + if not self.copier.contains(path): + # Add a reference to the manifest file in the parent manifest, if + # the manifest file is not a root manifest. + if entry.base: + parent = mozpath.dirname(entry.base) + relbase = mozpath.basename(entry.base) + relpath = mozpath.join(relbase, mozpath.basename(path)) + self.add_manifest(Manifest(parent, relpath)) + self.copier.add(path, ManifestFile(entry.base)) + + if isinstance(entry, ManifestChrome): + data = self._chrome_db.setdefault(entry.name, {}) + if isinstance(entry, ManifestMultiContent): + entries = data.setdefault(entry.type, {}).setdefault(entry.id, []) + else: + entries = data.setdefault(entry.type, []) + for e in entries: + # Ideally, we'd actually check whether entry.flags are more + # specific than e.flags, but in practice the following test + # is enough for now. + if entry == e: + errors.warn('"%s" is duplicated. Skipping.' % entry) + return + if not entry.flags or e.flags and entry.flags == e.flags: + errors.fatal('"%s" overrides "%s"' % (entry, e)) + entries.append(entry) + + self.copier[path].add(entry) + + def add_interfaces(self, path, content): + self.copier.add(path, content) + self.add_manifest( + ManifestInterfaces(mozpath.dirname(path), mozpath.basename(path)) + ) + + def contains(self, path): + assert "*" not in path + return self.copier.contains(path) + + +class JarFormatter(PiecemealFormatter): + """ + Formatter for the jar package format. Assumes manifest entries related to + chrome are registered before the chrome data files are added. Also assumes + manifest entries for resources are registered after chrome manifest + entries. + """ + + def __init__(self, copier, compress=True): + PiecemealFormatter.__init__(self, copier) + self._compress = compress + + def _add_base(self, base, addon=False): + if addon is True: + jarrer = Jarrer(self._compress) + self.copier.add(base + ".xpi", jarrer) + self._sub_formatter[base] = FlatSubFormatter(jarrer) + else: + self._sub_formatter[base] = JarSubFormatter( + FileRegistrySubtree(base, self.copier), self._compress + ) + + +class JarSubFormatter(PiecemealFormatter): + """ + Sub-formatter for the jar package format. It is a PiecemealFormatter that + dispatches between further sub-formatter for each of the jar files it + dispatches the chrome data to, and a FlatSubFormatter for the non-chrome + files. + """ + + def __init__(self, copier, compress=True): + PiecemealFormatter.__init__(self, copier) + self._frozen_chrome = False + self._compress = compress + self._sub_formatter[""] = FlatSubFormatter(copier) + + def _jarize(self, entry, relpath): + """ + Transform a manifest entry in one pointing to chrome data in a jar. + Return the corresponding chrome path and the new entry. + """ + base = entry.base + basepath = mozpath.split(relpath)[0] + chromepath = mozpath.join(base, basepath) + entry = ( + entry.rebase(chromepath) + .move(mozpath.join(base, "jar:%s.jar!" % basepath)) + .rebase(base) + ) + return chromepath, entry + + def add_manifest(self, entry): + if isinstance(entry, ManifestChrome) and not urlparse(entry.relpath).scheme: + chromepath, entry = self._jarize(entry, entry.relpath) + assert not self._frozen_chrome + if chromepath not in self._sub_formatter: + jarrer = Jarrer(self._compress) + self.copier.add(chromepath + ".jar", jarrer) + self._sub_formatter[chromepath] = FlatSubFormatter(jarrer) + elif isinstance(entry, ManifestResource) and not urlparse(entry.target).scheme: + chromepath, new_entry = self._jarize(entry, entry.target) + if chromepath in self._sub_formatter: + entry = new_entry + PiecemealFormatter.add_manifest(self, entry) + + +class OmniJarFormatter(JarFormatter): + """ + Formatter for the omnijar package format. + """ + + def __init__(self, copier, omnijar_name, compress=True, non_resources=()): + JarFormatter.__init__(self, copier, compress) + self._omnijar_name = omnijar_name + self._non_resources = non_resources + + def _add_base(self, base, addon=False): + if addon: + # Because add_base is always called with parents before children, + # all the possible ancestry of `base` is already present in + # `_sub_formatter`. + parent_base = mozpath.basedir(base, self._sub_formatter.keys()) + rel_base = mozpath.relpath(base, parent_base) + # If the addon is under a resource directory, package it in the + # omnijar. + parent_sub_formatter = self._sub_formatter[parent_base] + if parent_sub_formatter.is_resource(rel_base): + omnijar_sub_formatter = parent_sub_formatter._sub_formatter[ + self._omnijar_name + ] + self._sub_formatter[base] = FlatSubFormatter( + FileRegistrySubtree(rel_base, omnijar_sub_formatter.copier) + ) + return + JarFormatter._add_base(self, base, addon) + else: + self._sub_formatter[base] = OmniJarSubFormatter( + FileRegistrySubtree(base, self.copier), + self._omnijar_name, + self._compress, + self._non_resources, + ) + + +class OmniJarSubFormatter(PiecemealFormatter): + """ + Sub-formatter for the omnijar package format. It is a PiecemealFormatter + that dispatches between a FlatSubFormatter for the resources data and + another FlatSubFormatter for the other files. + """ + + def __init__(self, copier, omnijar_name, compress=True, non_resources=()): + PiecemealFormatter.__init__(self, copier) + self._omnijar_name = omnijar_name + self._compress = compress + self._non_resources = non_resources + self._sub_formatter[""] = FlatSubFormatter(copier) + jarrer = Jarrer(self._compress) + self._sub_formatter[omnijar_name] = FlatSubFormatter(jarrer) + + def _get_base(self, path): + base = self._omnijar_name if self.is_resource(path) else "" + # Only add the omnijar file if something ends up in it. + if base and not self.copier.contains(base): + self.copier.add(base, self._sub_formatter[base].copier) + return base, path + + def add_manifest(self, entry): + base = "" + if not isinstance(entry, ManifestBinaryComponent): + base = self._omnijar_name + formatter = self._sub_formatter[base] + return formatter.add_manifest(entry) + + def is_resource(self, path): + """ + Return whether the given path corresponds to a resource to be put in an + omnijar archive. + """ + if any(mozpath.match(path, p.replace("*", "**")) for p in self._non_resources): + return False + path = mozpath.split(path) + if path[0] == "chrome": + return len(path) == 1 or path[1] != "icons" + if path[0] == "components": + return path[-1].endswith((".js", ".xpt")) + if path[0] == "res": + return len(path) == 1 or ( + path[1] != "cursors" + and path[1] != "touchbar" + and path[1] != "MainMenu.nib" + ) + if path[0] == "defaults": + return len(path) != 3 or not ( + path[2] == "channel-prefs.js" and path[1] in ["pref", "preferences"] + ) + if len(path) <= 2 and path[-1] == "greprefs.js": + # Accommodate `greprefs.js` and `$ANDROID_CPU_ARCH/greprefs.js`. + return True + return path[0] in [ + "modules", + "actors", + "dictionaries", + "hyphenation", + "localization", + "update.locale", + "contentaccessible", + ] diff --git a/python/mozbuild/mozpack/packager/l10n.py b/python/mozbuild/mozpack/packager/l10n.py new file mode 100644 index 0000000000..76871e15cd --- /dev/null +++ b/python/mozbuild/mozpack/packager/l10n.py @@ -0,0 +1,304 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Replace localized parts of a packaged directory with data from a langpack +directory. +""" + +import json +import os + +import six +from createprecomplete import generate_precomplete + +import mozpack.path as mozpath +from mozpack.chrome.manifest import ( + Manifest, + ManifestChrome, + ManifestEntryWithRelPath, + ManifestLocale, + is_manifest, +) +from mozpack.copier import FileCopier, Jarrer +from mozpack.errors import errors +from mozpack.files import ComposedFinder, GeneratedFile, ManifestFile +from mozpack.mozjar import JAR_DEFLATED +from mozpack.packager import Component, SimpleManifestSink, SimplePackager +from mozpack.packager.formats import FlatFormatter, JarFormatter, OmniJarFormatter +from mozpack.packager.unpack import UnpackFinder + + +class LocaleManifestFinder(object): + def __init__(self, finder): + entries = self.entries = [] + bases = self.bases = [] + + class MockFormatter(object): + def add_interfaces(self, path, content): + pass + + def add(self, path, content): + pass + + def add_manifest(self, entry): + if entry.localized: + entries.append(entry) + + def add_base(self, base, addon=False): + bases.append(base) + + # SimplePackager rejects "manifest foo.manifest" entries with + # additional flags (such as "manifest foo.manifest application=bar"). + # Those type of entries are used by language packs to work as addons, + # but are not necessary for the purpose of l10n repacking. So we wrap + # the finder in order to remove those entries. + class WrapFinder(object): + def __init__(self, finder): + self._finder = finder + + def find(self, pattern): + for p, f in self._finder.find(pattern): + if isinstance(f, ManifestFile): + unwanted = [ + e for e in f._entries if isinstance(e, Manifest) and e.flags + ] + if unwanted: + f = ManifestFile( + f._base, [e for e in f._entries if e not in unwanted] + ) + yield p, f + + sink = SimpleManifestSink(WrapFinder(finder), MockFormatter()) + sink.add(Component(""), "*") + sink.close(False) + + # Find unique locales used in these manifest entries. + self.locales = list( + set(e.id for e in self.entries if isinstance(e, ManifestLocale)) + ) + + +class L10NRepackFormatterMixin(object): + def __init__(self, *args, **kwargs): + super(L10NRepackFormatterMixin, self).__init__(*args, **kwargs) + self._dictionaries = {} + + def add(self, path, file): + base, relpath = self._get_base(path) + if path.endswith(".dic"): + if relpath.startswith("dictionaries/"): + root, ext = mozpath.splitext(mozpath.basename(path)) + self._dictionaries[root] = path + elif path.endswith("/built_in_addons.json"): + data = json.loads(six.ensure_text(file.open().read())) + data["dictionaries"] = self._dictionaries + # The GeneratedFile content is only really generated after + # all calls to formatter.add. + file = GeneratedFile(lambda: json.dumps(data)) + elif relpath.startswith("META-INF/"): + # Ignore signatures inside omnijars. We drop these items: if we + # don't treat them as omnijar resources, they will be included in + # the top-level package, and that's not how omnijars are signed (Bug + # 1750676). If we treat them as omnijar resources, they will stay + # in the omnijar, as expected -- but the signatures won't be valid + # after repacking. Therefore, drop them. + return + super(L10NRepackFormatterMixin, self).add(path, file) + + +def L10NRepackFormatter(klass): + class L10NRepackFormatter(L10NRepackFormatterMixin, klass): + pass + + return L10NRepackFormatter + + +FlatFormatter = L10NRepackFormatter(FlatFormatter) +JarFormatter = L10NRepackFormatter(JarFormatter) +OmniJarFormatter = L10NRepackFormatter(OmniJarFormatter) + + +def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()): + app = LocaleManifestFinder(app_finder) + l10n = LocaleManifestFinder(l10n_finder) + + # The code further below assumes there's only one locale replaced with + # another one. + if len(app.locales) > 1: + errors.fatal("Multiple app locales aren't supported: " + ",".join(app.locales)) + if len(l10n.locales) > 1: + errors.fatal( + "Multiple l10n locales aren't supported: " + ",".join(l10n.locales) + ) + locale = app.locales[0] + l10n_locale = l10n.locales[0] + + # For each base directory, store what path a locale chrome package name + # corresponds to. + # e.g., for the following entry under app/chrome: + # locale foo en-US path/to/files + # keep track that the locale path for foo in app is + # app/chrome/path/to/files. + # As there may be multiple locale entries with the same base, but with + # different flags, that tracking takes the flags into account when there + # are some. Example: + # locale foo en-US path/to/files/win os=Win + # locale foo en-US path/to/files/mac os=Darwin + def key(entry): + if entry.flags: + return "%s %s" % (entry.name, entry.flags) + return entry.name + + l10n_paths = {} + for e in l10n.entries: + if isinstance(e, ManifestChrome): + base = mozpath.basedir(e.path, app.bases) + l10n_paths.setdefault(base, {}) + l10n_paths[base][key(e)] = e.path + + # For chrome and non chrome files or directories, store what langpack path + # corresponds to a package path. + paths = {} + for e in app.entries: + if isinstance(e, ManifestEntryWithRelPath): + base = mozpath.basedir(e.path, app.bases) + if base not in l10n_paths: + errors.fatal("Locale doesn't contain %s/" % base) + # Allow errors to accumulate + continue + if key(e) not in l10n_paths[base]: + errors.fatal("Locale doesn't have a manifest entry for '%s'" % e.name) + # Allow errors to accumulate + continue + paths[e.path] = l10n_paths[base][key(e)] + + for pattern in non_chrome: + for base in app.bases: + path = mozpath.join(base, pattern) + left = set(p for p, f in app_finder.find(path)) + right = set(p for p, f in l10n_finder.find(path)) + for p in right: + paths[p] = p + for p in left - right: + paths[p] = None + + # Create a new package, with non localized bits coming from the original + # package, and localized bits coming from the langpack. + packager = SimplePackager(formatter) + for p, f in app_finder: + if is_manifest(p): + # Remove localized manifest entries. + for e in [e for e in f if e.localized]: + f.remove(e) + # If the path is one that needs a locale replacement, use the + # corresponding file from the langpack. + path = None + if p in paths: + path = paths[p] + if not path: + continue + else: + base = mozpath.basedir(p, paths.keys()) + if base: + subpath = mozpath.relpath(p, base) + path = mozpath.normpath(mozpath.join(paths[base], subpath)) + + if path: + files = [f for p, f in l10n_finder.find(path)] + if not len(files): + if base not in non_chrome: + finderBase = "" + if hasattr(l10n_finder, "base"): + finderBase = l10n_finder.base + errors.error("Missing file: %s" % os.path.join(finderBase, path)) + else: + packager.add(path, files[0]) + else: + packager.add(p, f) + + # Add localized manifest entries from the langpack. + l10n_manifests = [] + for base in set(e.base for e in l10n.entries): + m = ManifestFile(base, [e for e in l10n.entries if e.base == base]) + path = mozpath.join(base, "chrome.%s.manifest" % l10n_locale) + l10n_manifests.append((path, m)) + bases = packager.get_bases() + for path, m in l10n_manifests: + base = mozpath.basedir(path, bases) + packager.add(path, m) + # Add a "manifest $path" entry in the top manifest under that base. + m = ManifestFile(base) + m.add(Manifest(base, mozpath.relpath(path, base))) + packager.add(mozpath.join(base, "chrome.manifest"), m) + + packager.close() + + # Add any remaining non chrome files. + for pattern in non_chrome: + for base in bases: + for p, f in l10n_finder.find(mozpath.join(base, pattern)): + if not formatter.contains(p): + formatter.add(p, f) + + # Resources in `localization` directories are packaged from the source and then + # if localized versions are present in the l10n dir, we package them as well + # keeping the source dir resources as a runtime fallback. + for p, f in l10n_finder.find("**/localization"): + if not formatter.contains(p): + formatter.add(p, f) + + # Transplant jar preloading information. + for path, log in six.iteritems(app_finder.jarlogs): + assert isinstance(copier[path], Jarrer) + copier[path].preload([l.replace(locale, l10n_locale) for l in log]) + + +def repack( + source, l10n, extra_l10n={}, non_resources=[], non_chrome=set(), minify=False +): + """ + Replace localized data from the `source` directory with localized data + from `l10n` and `extra_l10n`. + + The `source` argument points to a directory containing a packaged + application (in omnijar, jar or flat form). + The `l10n` argument points to a directory containing the main localized + data (usually in the form of a language pack addon) to use to replace + in the packaged application. + The `extra_l10n` argument contains a dict associating relative paths in + the source to separate directories containing localized data for them. + This can be used to point at different language pack addons for different + parts of the package application. + The `non_resources` argument gives a list of relative paths in the source + that should not be added in an omnijar in case the packaged application + is in that format. + The `non_chrome` argument gives a list of file/directory patterns for + localized files that are not listed in a chrome.manifest. + If `minify`, `.properties` files are minified. + """ + app_finder = UnpackFinder(source, minify=minify) + l10n_finder = UnpackFinder(l10n, minify=minify) + if extra_l10n: + finders = { + "": l10n_finder, + } + for base, path in six.iteritems(extra_l10n): + finders[base] = UnpackFinder(path, minify=minify) + l10n_finder = ComposedFinder(finders) + copier = FileCopier() + compress = min(app_finder.compressed, JAR_DEFLATED) + if app_finder.kind == "flat": + formatter = FlatFormatter(copier) + elif app_finder.kind == "jar": + formatter = JarFormatter(copier, compress=compress) + elif app_finder.kind == "omni": + formatter = OmniJarFormatter( + copier, app_finder.omnijar, compress=compress, non_resources=non_resources + ) + + with errors.accumulate(): + _repack(app_finder, l10n_finder, copier, formatter, non_chrome) + copier.copy(source, skip_if_older=False) + generate_precomplete(source) diff --git a/python/mozbuild/mozpack/packager/unpack.py b/python/mozbuild/mozpack/packager/unpack.py new file mode 100644 index 0000000000..dff295eb9b --- /dev/null +++ b/python/mozbuild/mozpack/packager/unpack.py @@ -0,0 +1,200 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import codecs + +from six.moves.urllib.parse import urlparse + +import mozpack.path as mozpath +from mozpack.chrome.manifest import ( + ManifestEntryWithRelPath, + ManifestResource, + is_manifest, + parse_manifest, +) +from mozpack.copier import FileCopier, FileRegistry +from mozpack.files import BaseFinder, DeflatedFile, FileFinder, ManifestFile +from mozpack.mozjar import JarReader +from mozpack.packager import SimplePackager +from mozpack.packager.formats import FlatFormatter + + +class UnpackFinder(BaseFinder): + """ + Special Finder object that treats the source package directory as if it + were in the flat chrome format, whatever chrome format it actually is in. + + This means that for example, paths like chrome/browser/content/... match + files under jar:chrome/browser.jar!/content/... in case of jar chrome + format. + + The only argument to the constructor is a Finder instance or a path. + The UnpackFinder is populated with files from this Finder instance, + or with files from a FileFinder using the given path as its root. + """ + + def __init__(self, source, omnijar_name=None, unpack_xpi=True, **kwargs): + if isinstance(source, BaseFinder): + assert not kwargs + self._finder = source + else: + self._finder = FileFinder(source, **kwargs) + self.base = self._finder.base + self.files = FileRegistry() + self.kind = "flat" + if omnijar_name: + self.omnijar = omnijar_name + else: + # Can't include globally because of bootstrapping issues. + from buildconfig import substs + + self.omnijar = substs.get("OMNIJAR_NAME", "omni.ja") + self.jarlogs = {} + self.compressed = False + self._unpack_xpi = unpack_xpi + + jars = set() + + for p, f in self._finder.find("*"): + # Skip the precomplete file, which is generated at packaging time. + if p == "precomplete": + continue + base = mozpath.dirname(p) + # If the file matches the omnijar pattern, it is an omnijar. + # All the files it contains go in the directory containing the full + # pattern. Manifests are merged if there is a corresponding manifest + # in the directory. + if self._maybe_zip(f) and mozpath.match(p, "**/%s" % self.omnijar): + jar = self._open_jar(p, f) + if "chrome.manifest" in jar: + self.kind = "omni" + self._fill_with_jar(p[: -len(self.omnijar) - 1], jar) + continue + # If the file is a manifest, scan its entries for some referencing + # jar: urls. If there are some, the files contained in the jar they + # point to, go under a directory named after the jar. + if is_manifest(p): + m = self.files[p] if self.files.contains(p) else ManifestFile(base) + for e in parse_manifest( + self.base, p, codecs.getreader("utf-8")(f.open()) + ): + m.add(self._handle_manifest_entry(e, jars)) + if self.files.contains(p): + continue + f = m + # If we're unpacking packed addons and the file is a packed addon, + # unpack it under a directory named after the xpi. + if self._unpack_xpi and p.endswith(".xpi") and self._maybe_zip(f): + self._fill_with_jar(p[:-4], self._open_jar(p, f)) + continue + if p not in jars: + self.files.add(p, f) + + def _fill_with_jar(self, base, jar): + for j in jar: + path = mozpath.join(base, j.filename) + if is_manifest(j.filename): + m = ( + self.files[path] + if self.files.contains(path) + else ManifestFile(mozpath.dirname(path)) + ) + for e in parse_manifest(None, path, j): + m.add(e) + if not self.files.contains(path): + self.files.add(path, m) + continue + else: + self.files.add(path, DeflatedFile(j)) + + def _handle_manifest_entry(self, entry, jars): + jarpath = None + if ( + isinstance(entry, ManifestEntryWithRelPath) + and urlparse(entry.relpath).scheme == "jar" + ): + jarpath, entry = self._unjarize(entry, entry.relpath) + elif ( + isinstance(entry, ManifestResource) + and urlparse(entry.target).scheme == "jar" + ): + jarpath, entry = self._unjarize(entry, entry.target) + if jarpath: + # Don't defer unpacking the jar file. If we already saw + # it, take (and remove) it from the registry. If we + # haven't, try to find it now. + if self.files.contains(jarpath): + jar = self.files[jarpath] + self.files.remove(jarpath) + else: + jar = [f for p, f in self._finder.find(jarpath)] + assert len(jar) == 1 + jar = jar[0] + if jarpath not in jars: + base = mozpath.splitext(jarpath)[0] + for j in self._open_jar(jarpath, jar): + self.files.add(mozpath.join(base, j.filename), DeflatedFile(j)) + jars.add(jarpath) + self.kind = "jar" + return entry + + def _open_jar(self, path, file): + """ + Return a JarReader for the given BaseFile instance, keeping a log of + the preloaded entries it has. + """ + jar = JarReader(fileobj=file.open()) + self.compressed = max(self.compressed, jar.compression) + if jar.last_preloaded: + jarlog = list(jar.entries.keys()) + self.jarlogs[path] = jarlog[: jarlog.index(jar.last_preloaded) + 1] + return jar + + def find(self, path): + for p in self.files.match(path): + yield p, self.files[p] + + def _maybe_zip(self, file): + """ + Return whether the given BaseFile looks like a ZIP/Jar. + """ + header = file.open().read(8) + return len(header) == 8 and (header[0:2] == b"PK" or header[4:6] == b"PK") + + def _unjarize(self, entry, relpath): + """ + Transform a manifest entry pointing to chrome data in a jar in one + pointing to the corresponding unpacked path. Return the jar path and + the new entry. + """ + base = entry.base + jar, relpath = urlparse(relpath).path.split("!", 1) + entry = ( + entry.rebase(mozpath.join(base, "jar:%s!" % jar)) + .move(mozpath.join(base, mozpath.splitext(jar)[0])) + .rebase(base) + ) + return mozpath.join(base, jar), entry + + +def unpack_to_registry(source, registry, omnijar_name=None): + """ + Transform a jar chrome or omnijar packaged directory into a flat package. + + The given registry is filled with the flat package. + """ + finder = UnpackFinder(source, omnijar_name) + packager = SimplePackager(FlatFormatter(registry)) + for p, f in finder.find("*"): + packager.add(p, f) + packager.close() + + +def unpack(source, omnijar_name=None): + """ + Transform a jar chrome or omnijar packaged directory into a flat package. + """ + copier = FileCopier() + unpack_to_registry(source, copier, omnijar_name) + copier.copy(source, skip_if_older=False) |