summaryrefslogtreecommitdiffstats
path: root/python/mozbuild/mozpack/packager/l10n.py
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--python/mozbuild/mozpack/packager/l10n.py304
1 files changed, 304 insertions, 0 deletions
diff --git a/python/mozbuild/mozpack/packager/l10n.py b/python/mozbuild/mozpack/packager/l10n.py
new file mode 100644
index 0000000000..76871e15cd
--- /dev/null
+++ b/python/mozbuild/mozpack/packager/l10n.py
@@ -0,0 +1,304 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Replace localized parts of a packaged directory with data from a langpack
+directory.
+"""
+
+import json
+import os
+
+import six
+from createprecomplete import generate_precomplete
+
+import mozpack.path as mozpath
+from mozpack.chrome.manifest import (
+ Manifest,
+ ManifestChrome,
+ ManifestEntryWithRelPath,
+ ManifestLocale,
+ is_manifest,
+)
+from mozpack.copier import FileCopier, Jarrer
+from mozpack.errors import errors
+from mozpack.files import ComposedFinder, GeneratedFile, ManifestFile
+from mozpack.mozjar import JAR_DEFLATED
+from mozpack.packager import Component, SimpleManifestSink, SimplePackager
+from mozpack.packager.formats import FlatFormatter, JarFormatter, OmniJarFormatter
+from mozpack.packager.unpack import UnpackFinder
+
+
+class LocaleManifestFinder(object):
+ def __init__(self, finder):
+ entries = self.entries = []
+ bases = self.bases = []
+
+ class MockFormatter(object):
+ def add_interfaces(self, path, content):
+ pass
+
+ def add(self, path, content):
+ pass
+
+ def add_manifest(self, entry):
+ if entry.localized:
+ entries.append(entry)
+
+ def add_base(self, base, addon=False):
+ bases.append(base)
+
+ # SimplePackager rejects "manifest foo.manifest" entries with
+ # additional flags (such as "manifest foo.manifest application=bar").
+ # Those type of entries are used by language packs to work as addons,
+ # but are not necessary for the purpose of l10n repacking. So we wrap
+ # the finder in order to remove those entries.
+ class WrapFinder(object):
+ def __init__(self, finder):
+ self._finder = finder
+
+ def find(self, pattern):
+ for p, f in self._finder.find(pattern):
+ if isinstance(f, ManifestFile):
+ unwanted = [
+ e for e in f._entries if isinstance(e, Manifest) and e.flags
+ ]
+ if unwanted:
+ f = ManifestFile(
+ f._base, [e for e in f._entries if e not in unwanted]
+ )
+ yield p, f
+
+ sink = SimpleManifestSink(WrapFinder(finder), MockFormatter())
+ sink.add(Component(""), "*")
+ sink.close(False)
+
+ # Find unique locales used in these manifest entries.
+ self.locales = list(
+ set(e.id for e in self.entries if isinstance(e, ManifestLocale))
+ )
+
+
+class L10NRepackFormatterMixin(object):
+ def __init__(self, *args, **kwargs):
+ super(L10NRepackFormatterMixin, self).__init__(*args, **kwargs)
+ self._dictionaries = {}
+
+ def add(self, path, file):
+ base, relpath = self._get_base(path)
+ if path.endswith(".dic"):
+ if relpath.startswith("dictionaries/"):
+ root, ext = mozpath.splitext(mozpath.basename(path))
+ self._dictionaries[root] = path
+ elif path.endswith("/built_in_addons.json"):
+ data = json.loads(six.ensure_text(file.open().read()))
+ data["dictionaries"] = self._dictionaries
+ # The GeneratedFile content is only really generated after
+ # all calls to formatter.add.
+ file = GeneratedFile(lambda: json.dumps(data))
+ elif relpath.startswith("META-INF/"):
+ # Ignore signatures inside omnijars. We drop these items: if we
+ # don't treat them as omnijar resources, they will be included in
+ # the top-level package, and that's not how omnijars are signed (Bug
+ # 1750676). If we treat them as omnijar resources, they will stay
+ # in the omnijar, as expected -- but the signatures won't be valid
+ # after repacking. Therefore, drop them.
+ return
+ super(L10NRepackFormatterMixin, self).add(path, file)
+
+
+def L10NRepackFormatter(klass):
+ class L10NRepackFormatter(L10NRepackFormatterMixin, klass):
+ pass
+
+ return L10NRepackFormatter
+
+
+FlatFormatter = L10NRepackFormatter(FlatFormatter)
+JarFormatter = L10NRepackFormatter(JarFormatter)
+OmniJarFormatter = L10NRepackFormatter(OmniJarFormatter)
+
+
+def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()):
+ app = LocaleManifestFinder(app_finder)
+ l10n = LocaleManifestFinder(l10n_finder)
+
+ # The code further below assumes there's only one locale replaced with
+ # another one.
+ if len(app.locales) > 1:
+ errors.fatal("Multiple app locales aren't supported: " + ",".join(app.locales))
+ if len(l10n.locales) > 1:
+ errors.fatal(
+ "Multiple l10n locales aren't supported: " + ",".join(l10n.locales)
+ )
+ locale = app.locales[0]
+ l10n_locale = l10n.locales[0]
+
+ # For each base directory, store what path a locale chrome package name
+ # corresponds to.
+ # e.g., for the following entry under app/chrome:
+ # locale foo en-US path/to/files
+ # keep track that the locale path for foo in app is
+ # app/chrome/path/to/files.
+ # As there may be multiple locale entries with the same base, but with
+ # different flags, that tracking takes the flags into account when there
+ # are some. Example:
+ # locale foo en-US path/to/files/win os=Win
+ # locale foo en-US path/to/files/mac os=Darwin
+ def key(entry):
+ if entry.flags:
+ return "%s %s" % (entry.name, entry.flags)
+ return entry.name
+
+ l10n_paths = {}
+ for e in l10n.entries:
+ if isinstance(e, ManifestChrome):
+ base = mozpath.basedir(e.path, app.bases)
+ l10n_paths.setdefault(base, {})
+ l10n_paths[base][key(e)] = e.path
+
+ # For chrome and non chrome files or directories, store what langpack path
+ # corresponds to a package path.
+ paths = {}
+ for e in app.entries:
+ if isinstance(e, ManifestEntryWithRelPath):
+ base = mozpath.basedir(e.path, app.bases)
+ if base not in l10n_paths:
+ errors.fatal("Locale doesn't contain %s/" % base)
+ # Allow errors to accumulate
+ continue
+ if key(e) not in l10n_paths[base]:
+ errors.fatal("Locale doesn't have a manifest entry for '%s'" % e.name)
+ # Allow errors to accumulate
+ continue
+ paths[e.path] = l10n_paths[base][key(e)]
+
+ for pattern in non_chrome:
+ for base in app.bases:
+ path = mozpath.join(base, pattern)
+ left = set(p for p, f in app_finder.find(path))
+ right = set(p for p, f in l10n_finder.find(path))
+ for p in right:
+ paths[p] = p
+ for p in left - right:
+ paths[p] = None
+
+ # Create a new package, with non localized bits coming from the original
+ # package, and localized bits coming from the langpack.
+ packager = SimplePackager(formatter)
+ for p, f in app_finder:
+ if is_manifest(p):
+ # Remove localized manifest entries.
+ for e in [e for e in f if e.localized]:
+ f.remove(e)
+ # If the path is one that needs a locale replacement, use the
+ # corresponding file from the langpack.
+ path = None
+ if p in paths:
+ path = paths[p]
+ if not path:
+ continue
+ else:
+ base = mozpath.basedir(p, paths.keys())
+ if base:
+ subpath = mozpath.relpath(p, base)
+ path = mozpath.normpath(mozpath.join(paths[base], subpath))
+
+ if path:
+ files = [f for p, f in l10n_finder.find(path)]
+ if not len(files):
+ if base not in non_chrome:
+ finderBase = ""
+ if hasattr(l10n_finder, "base"):
+ finderBase = l10n_finder.base
+ errors.error("Missing file: %s" % os.path.join(finderBase, path))
+ else:
+ packager.add(path, files[0])
+ else:
+ packager.add(p, f)
+
+ # Add localized manifest entries from the langpack.
+ l10n_manifests = []
+ for base in set(e.base for e in l10n.entries):
+ m = ManifestFile(base, [e for e in l10n.entries if e.base == base])
+ path = mozpath.join(base, "chrome.%s.manifest" % l10n_locale)
+ l10n_manifests.append((path, m))
+ bases = packager.get_bases()
+ for path, m in l10n_manifests:
+ base = mozpath.basedir(path, bases)
+ packager.add(path, m)
+ # Add a "manifest $path" entry in the top manifest under that base.
+ m = ManifestFile(base)
+ m.add(Manifest(base, mozpath.relpath(path, base)))
+ packager.add(mozpath.join(base, "chrome.manifest"), m)
+
+ packager.close()
+
+ # Add any remaining non chrome files.
+ for pattern in non_chrome:
+ for base in bases:
+ for p, f in l10n_finder.find(mozpath.join(base, pattern)):
+ if not formatter.contains(p):
+ formatter.add(p, f)
+
+ # Resources in `localization` directories are packaged from the source and then
+ # if localized versions are present in the l10n dir, we package them as well
+ # keeping the source dir resources as a runtime fallback.
+ for p, f in l10n_finder.find("**/localization"):
+ if not formatter.contains(p):
+ formatter.add(p, f)
+
+ # Transplant jar preloading information.
+ for path, log in six.iteritems(app_finder.jarlogs):
+ assert isinstance(copier[path], Jarrer)
+ copier[path].preload([l.replace(locale, l10n_locale) for l in log])
+
+
+def repack(
+ source, l10n, extra_l10n={}, non_resources=[], non_chrome=set(), minify=False
+):
+ """
+ Replace localized data from the `source` directory with localized data
+ from `l10n` and `extra_l10n`.
+
+ The `source` argument points to a directory containing a packaged
+ application (in omnijar, jar or flat form).
+ The `l10n` argument points to a directory containing the main localized
+ data (usually in the form of a language pack addon) to use to replace
+ in the packaged application.
+ The `extra_l10n` argument contains a dict associating relative paths in
+ the source to separate directories containing localized data for them.
+ This can be used to point at different language pack addons for different
+ parts of the package application.
+ The `non_resources` argument gives a list of relative paths in the source
+ that should not be added in an omnijar in case the packaged application
+ is in that format.
+ The `non_chrome` argument gives a list of file/directory patterns for
+ localized files that are not listed in a chrome.manifest.
+ If `minify`, `.properties` files are minified.
+ """
+ app_finder = UnpackFinder(source, minify=minify)
+ l10n_finder = UnpackFinder(l10n, minify=minify)
+ if extra_l10n:
+ finders = {
+ "": l10n_finder,
+ }
+ for base, path in six.iteritems(extra_l10n):
+ finders[base] = UnpackFinder(path, minify=minify)
+ l10n_finder = ComposedFinder(finders)
+ copier = FileCopier()
+ compress = min(app_finder.compressed, JAR_DEFLATED)
+ if app_finder.kind == "flat":
+ formatter = FlatFormatter(copier)
+ elif app_finder.kind == "jar":
+ formatter = JarFormatter(copier, compress=compress)
+ elif app_finder.kind == "omni":
+ formatter = OmniJarFormatter(
+ copier, app_finder.omnijar, compress=compress, non_resources=non_resources
+ )
+
+ with errors.accumulate():
+ _repack(app_finder, l10n_finder, copier, formatter, non_chrome)
+ copier.copy(source, skip_if_older=False)
+ generate_precomplete(source)