diff options
Diffstat (limited to 'python/mozbuild/mozbuild/repackaging/deb.py')
| -rw-r--r-- | python/mozbuild/mozbuild/repackaging/deb.py | 694 |
1 files changed, 694 insertions, 0 deletions
diff --git a/python/mozbuild/mozbuild/repackaging/deb.py b/python/mozbuild/mozbuild/repackaging/deb.py new file mode 100644 index 0000000000..3e01680437 --- /dev/null +++ b/python/mozbuild/mozbuild/repackaging/deb.py @@ -0,0 +1,694 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import datetime +import json +import logging +import os +import shutil +import subprocess +import tarfile +import tempfile +import zipfile +from email.utils import format_datetime +from pathlib import Path +from string import Template + +import mozfile +import mozpack.path as mozpath +import requests +from mozilla_version.gecko import GeckoVersion +from redo import retry + +from mozbuild.repackaging.application_ini import get_application_ini_values + + +class NoDebPackageFound(Exception): + """Raised when no .deb is found after calling dpkg-buildpackage""" + + def __init__(self, deb_file_path) -> None: + super().__init__( + f"No {deb_file_path} package found after calling dpkg-buildpackage" + ) + + +class HgServerError(Exception): + """Raised when Hg responds with an error code that is not 404 (i.e. when there is an outage)""" + + def __init__(self, msg) -> None: + super().__init__(msg) + + +_DEB_ARCH = { + "all": "all", + "x86": "i386", + "x86_64": "amd64", +} +# At the moment the Firefox build baseline is jessie. +# The debian-repackage image defined in taskcluster/docker/debian-repackage/Dockerfile +# bootstraps the /srv/jessie-i386 and /srv/jessie-amd64 chroot environments we use to +# create the `.deb` repackages. By running the repackage using chroot we generate shared +# library dependencies that match the Firefox build baseline +# defined in taskcluster/scripts/misc/build-sysroot.sh +_DEB_DIST = "jessie" + + +def repackage_deb( + log, + infile, + output, + template_dir, + arch, + version, + build_number, + release_product, + release_type, + fluent_localization, + fluent_resource_loader, +): + if not tarfile.is_tarfile(infile): + raise Exception("Input file %s is not a valid tarfile." % infile) + + tmpdir = _create_temporary_directory(arch) + source_dir = os.path.join(tmpdir, "source") + try: + mozfile.extract_tarball(infile, source_dir) + application_ini_data = _extract_application_ini_data(infile) + build_variables = _get_build_variables( + application_ini_data, + arch, + version, + build_number, + depends="${shlibs:Depends},", + ) + + _copy_plain_deb_config(template_dir, source_dir) + _render_deb_templates(template_dir, source_dir, build_variables) + + app_name = application_ini_data["name"] + with open( + mozpath.join(source_dir, app_name.lower(), "is-packaged-app"), "w" + ) as f: + f.write("This is a packaged app.\n") + + _inject_deb_distribution_folder(source_dir, app_name) + _inject_deb_desktop_entry_file( + log, + source_dir, + build_variables, + release_product, + release_type, + fluent_localization, + fluent_resource_loader, + ) + _generate_deb_archive( + source_dir, + target_dir=tmpdir, + output_file_path=output, + build_variables=build_variables, + arch=arch, + ) + + finally: + shutil.rmtree(tmpdir) + + +def repackage_deb_l10n( + input_xpi_file, input_tar_file, output, template_dir, version, build_number +): + arch = "all" + + tmpdir = _create_temporary_directory(arch) + source_dir = os.path.join(tmpdir, "source") + try: + langpack_metadata = _extract_langpack_metadata(input_xpi_file) + langpack_dir = mozpath.join(source_dir, "firefox", "distribution", "extensions") + application_ini_data = _extract_application_ini_data(input_tar_file) + langpack_id = langpack_metadata["langpack_id"] + build_variables = _get_build_variables( + application_ini_data, + arch, + version, + build_number, + depends=application_ini_data["remoting_name"], + # Debian package names are only lowercase + package_name_suffix=f"-l10n-{langpack_id.lower()}", + description_suffix=f" - {langpack_metadata['description']}", + ) + _copy_plain_deb_config(template_dir, source_dir) + _render_deb_templates(template_dir, source_dir, build_variables) + + os.makedirs(langpack_dir, exist_ok=True) + shutil.copy( + input_xpi_file, + mozpath.join( + langpack_dir, + f"{langpack_metadata['browser_specific_settings']['gecko']['id']}.xpi", + ), + ) + _generate_deb_archive( + source_dir=source_dir, + target_dir=tmpdir, + output_file_path=output, + build_variables=build_variables, + arch=arch, + ) + finally: + shutil.rmtree(tmpdir) + + +def _extract_application_ini_data(input_tar_file): + with tempfile.TemporaryDirectory() as d: + with tarfile.open(input_tar_file) as tar: + application_ini_files = [ + tar_info + for tar_info in tar.getmembers() + if tar_info.name.endswith("/application.ini") + ] + if len(application_ini_files) == 0: + raise ValueError( + f"Cannot find any application.ini file in archive {input_tar_file}" + ) + if len(application_ini_files) > 1: + raise ValueError( + f"Too many application.ini files found in archive {input_tar_file}. " + f"Found: {application_ini_files}" + ) + + tar.extract(application_ini_files[0], path=d) + + return _extract_application_ini_data_from_directory(d) + + +def _extract_application_ini_data_from_directory(application_directory): + values = get_application_ini_values( + application_directory, + dict(section="App", value="Name"), + dict(section="App", value="CodeName", fallback="Name"), + dict(section="App", value="Vendor"), + dict(section="App", value="RemotingName"), + dict(section="App", value="BuildID"), + ) + + data = { + "name": next(values), + "display_name": next(values), + "vendor": next(values), + "remoting_name": next(values), + "build_id": next(values), + } + data["timestamp"] = datetime.datetime.strptime(data["build_id"], "%Y%m%d%H%M%S") + + return data + + +def _get_build_variables( + application_ini_data, + arch, + version_string, + build_number, + depends, + package_name_suffix="", + description_suffix="", +): + version = GeckoVersion.parse(version_string) + # Nightlies don't have build numbers + deb_pkg_version = ( + f"{version}~{application_ini_data['build_id']}" + if version.is_nightly + else f"{version}~build{build_number}" + ) + remoting_name = application_ini_data["remoting_name"].lower() + + return { + "DEB_DESCRIPTION": f"{application_ini_data['vendor']} {application_ini_data['display_name']}" + f"{description_suffix}", + "DEB_PKG_INSTALL_PATH": f"usr/lib/{remoting_name}", + "DEB_PKG_NAME": f"{remoting_name}{package_name_suffix}", + "DEB_PKG_VERSION": deb_pkg_version, + "DEB_CHANGELOG_DATE": format_datetime(application_ini_data["timestamp"]), + "DEB_ARCH_NAME": _DEB_ARCH[arch], + "DEB_DEPENDS": depends, + } + + +def _copy_plain_deb_config(input_template_dir, source_dir): + template_dir_filenames = os.listdir(input_template_dir) + plain_filenames = [ + mozpath.basename(filename) + for filename in template_dir_filenames + if not filename.endswith(".in") + ] + os.makedirs(mozpath.join(source_dir, "debian"), exist_ok=True) + + for filename in plain_filenames: + shutil.copy( + mozpath.join(input_template_dir, filename), + mozpath.join(source_dir, "debian", filename), + ) + + +def _render_deb_templates( + input_template_dir, source_dir, build_variables, exclude_file_names=None +): + exclude_file_names = [] if exclude_file_names is None else exclude_file_names + + template_dir_filenames = os.listdir(input_template_dir) + template_filenames = [ + mozpath.basename(filename) + for filename in template_dir_filenames + if filename.endswith(".in") and filename not in exclude_file_names + ] + os.makedirs(mozpath.join(source_dir, "debian"), exist_ok=True) + + for file_name in template_filenames: + with open(mozpath.join(input_template_dir, file_name)) as f: + template = Template(f.read()) + with open(mozpath.join(source_dir, "debian", Path(file_name).stem), "w") as f: + f.write(template.substitute(build_variables)) + + +def _inject_deb_distribution_folder(source_dir, app_name): + with tempfile.TemporaryDirectory() as git_clone_dir: + subprocess.check_call( + [ + "git", + "clone", + "https://github.com/mozilla-partners/deb.git", + git_clone_dir, + ], + ) + shutil.copytree( + mozpath.join(git_clone_dir, "desktop/deb/distribution"), + mozpath.join(source_dir, app_name.lower(), "distribution"), + ) + + +def _inject_deb_desktop_entry_file( + log, + source_dir, + build_variables, + release_product, + release_type, + fluent_localization, + fluent_resource_loader, +): + desktop_entry_file_text = _generate_browser_desktop_entry_file_text( + log, + build_variables, + release_product, + release_type, + fluent_localization, + fluent_resource_loader, + ) + desktop_entry_file_filename = f"{build_variables['DEB_PKG_NAME']}.desktop" + os.makedirs(mozpath.join(source_dir, "debian"), exist_ok=True) + with open( + mozpath.join(source_dir, "debian", desktop_entry_file_filename), "w" + ) as f: + f.write(desktop_entry_file_text) + + +def _generate_browser_desktop_entry_file_text( + log, + build_variables, + release_product, + release_type, + fluent_localization, + fluent_resource_loader, +): + localizations = _create_fluent_localizations( + fluent_resource_loader, fluent_localization, release_type, release_product, log + ) + desktop_entry = _generate_browser_desktop_entry(build_variables, localizations) + desktop_entry_file_text = "\n".join(desktop_entry) + return desktop_entry_file_text + + +def _create_fluent_localizations( + fluent_resource_loader, fluent_localization, release_type, release_product, log +): + brand_fluent_filename = "brand.ftl" + l10n_central_url = "https://hg.mozilla.org/l10n-central" + desktop_entry_fluent_filename = "linuxDesktopEntry.ftl" + + l10n_dir = tempfile.mkdtemp() + + loader = fluent_resource_loader(os.path.join(l10n_dir, "{locale}")) + + localizations = {} + linux_l10n_changesets = _load_linux_l10n_changesets( + "browser/locales/l10n-changesets.json" + ) + locales = ["en-US"] + locales.extend(linux_l10n_changesets.keys()) + en_US_brand_fluent_filename = _get_en_US_brand_fluent_filename( + brand_fluent_filename, release_type, release_product + ) + + for locale in locales: + locale_dir = os.path.join(l10n_dir, locale) + os.mkdir(locale_dir) + localized_desktop_entry_filename = os.path.join( + locale_dir, desktop_entry_fluent_filename + ) + if locale == "en-US": + en_US_desktop_entry_fluent_filename = os.path.join( + "browser/locales/en-US/browser", desktop_entry_fluent_filename + ) + shutil.copyfile( + en_US_desktop_entry_fluent_filename, + localized_desktop_entry_filename, + ) + else: + non_en_US_desktop_entry_fluent_filename = os.path.join( + "browser/browser", desktop_entry_fluent_filename + ) + non_en_US_fluent_resource_file_url = os.path.join( + l10n_central_url, + locale, + "raw-file", + linux_l10n_changesets[locale]["revision"], + non_en_US_desktop_entry_fluent_filename, + ) + response = requests.get(non_en_US_fluent_resource_file_url) + response = retry( + requests.get, + args=[non_en_US_fluent_resource_file_url], + attempts=5, + sleeptime=3, + jitter=2, + ) + mgs = "Missing {fluent_resource_file_name} for {locale}: received HTTP {status_code} for GET {resource_file_url}" + params = { + "fluent_resource_file_name": desktop_entry_fluent_filename, + "locale": locale, + "resource_file_url": non_en_US_fluent_resource_file_url, + "status_code": response.status_code, + } + action = "repackage-deb" + if response.status_code == 404: + log( + logging.WARNING, + action, + params, + mgs, + ) + continue + if response.status_code != 200: + log( + logging.ERROR, + action, + params, + mgs, + ) + raise HgServerError(mgs.format(**params)) + + with open(localized_desktop_entry_filename, "w", encoding="utf-8") as f: + f.write(response.text) + + shutil.copyfile( + en_US_brand_fluent_filename, + os.path.join(locale_dir, brand_fluent_filename), + ) + + fallbacks = [locale] + if locale != "en-US": + fallbacks.append("en-US") + localizations[locale] = fluent_localization( + fallbacks, [desktop_entry_fluent_filename, brand_fluent_filename], loader + ) + + return localizations + + +def _get_en_US_brand_fluent_filename( + brand_fluent_filename, release_type, release_product +): + branding_fluent_filename_template = os.path.join( + "browser/branding/{brand}/locales/en-US", brand_fluent_filename + ) + if release_type == "nightly": + return branding_fluent_filename_template.format(brand="nightly") + elif release_type == "beta" and release_product == "firefox": + return branding_fluent_filename_template.format(brand="official") + elif release_type == "beta" and release_product == "devedition": + return branding_fluent_filename_template.format(brand="aurora") + else: + return branding_fluent_filename_template.format(brand="unofficial") + + +def _load_linux_l10n_changesets(l10n_changesets_filename): + with open(l10n_changesets_filename) as l10n_changesets_file: + l10n_changesets = json.load(l10n_changesets_file) + return { + locale: changeset + for locale, changeset in l10n_changesets.items() + if any(platform.startswith("linux") for platform in changeset["platforms"]) + } + + +def _generate_browser_desktop_entry(build_variables, localizations): + mime_types = [ + "application/json", + "application/pdf", + "application/rdf+xml", + "application/rss+xml", + "application/x-xpinstall", + "application/xhtml+xml", + "application/xml", + "audio/flac", + "audio/ogg", + "audio/webm", + "image/avif", + "image/gif", + "image/jpeg", + "image/png", + "image/svg+xml", + "image/webp", + "text/html", + "text/xml", + "video/ogg", + "video/webm", + "x-scheme-handler/chrome", + "x-scheme-handler/http", + "x-scheme-handler/https", + ] + + categories = [ + "GNOME", + "GTK", + "Network", + "WebBrowser", + ] + + actions = [ + { + "name": "new-window", + "message": "desktop-action-new-window-name", + "command": f"{build_variables['DEB_PKG_NAME']} --new-window %u", + }, + { + "name": "new-private-window", + "message": "desktop-action-new-private-window-name", + "command": f"{build_variables['DEB_PKG_NAME']} --private-window %u", + }, + { + "name": "open-profile-manager", + "message": "desktop-action-open-profile-manager", + "command": f"{build_variables['DEB_PKG_NAME']} --ProfileManager", + }, + ] + + desktop_entry = _desktop_entry_section( + "Desktop Entry", + [ + { + "key": "Version", + "value": "1.0", + }, + { + "key": "Type", + "value": "Application", + }, + { + "key": "Exec", + "value": f"{build_variables['DEB_PKG_NAME']} %u", + }, + { + "key": "Terminal", + "value": "false", + }, + { + "key": "X-MultipleArgs", + "value": "false", + }, + { + "key": "Icon", + "value": build_variables["DEB_PKG_NAME"], + }, + { + "key": "StartupWMClass", + "value": build_variables["DEB_PKG_NAME"], + }, + { + "key": "Categories", + "value": _desktop_entry_list(categories), + }, + { + "key": "MimeType", + "value": _desktop_entry_list(mime_types), + }, + { + "key": "StartupNotify", + "value": "true", + }, + { + "key": "Actions", + "value": _desktop_entry_list([action["name"] for action in actions]), + }, + {"key": "Name", "value": "desktop-entry-name", "l10n": True}, + {"key": "Comment", "value": "desktop-entry-comment", "l10n": True}, + {"key": "GenericName", "value": "desktop-entry-generic-name", "l10n": True}, + {"key": "Keywords", "value": "desktop-entry-keywords", "l10n": True}, + { + "key": "X-GNOME-FullName", + "value": "desktop-entry-x-gnome-full-name", + "l10n": True, + }, + ], + localizations, + ) + + for action in actions: + desktop_entry.extend( + _desktop_entry_section( + f"Desktop Action {action['name']}", + [ + { + "key": "Name", + "value": action["message"], + "l10n": True, + }, + { + "key": "Exec", + "value": action["command"], + }, + ], + localizations, + ) + ) + + return desktop_entry + + +def _desktop_entry_list(iterable): + delimiter = ";" + return f"{delimiter.join(iterable)}{delimiter}" + + +def _desktop_entry_attribute(key, value, locale=None, localizations=None): + if not locale and not localizations: + return f"{key}={value}" + if locale and locale == "en-US": + return f"{key}={localizations[locale].format_value(value)}" + else: + return f"{key}[{locale.replace('-', '_')}]={localizations[locale].format_value(value)}" + + +def _desktop_entry_section(header, attributes, localizations): + desktop_entry_section = [f"[{header}]"] + l10n_attributes = [attribute for attribute in attributes if attribute.get("l10n")] + non_l10n_attributes = [ + attribute for attribute in attributes if not attribute.get("l10n") + ] + for attribute in non_l10n_attributes: + desktop_entry_section.append( + _desktop_entry_attribute(attribute["key"], attribute["value"]) + ) + for attribute in l10n_attributes: + for locale in localizations: + desktop_entry_section.append( + _desktop_entry_attribute( + attribute["key"], attribute["value"], locale, localizations + ) + ) + desktop_entry_section.append("") + return desktop_entry_section + + +def _generate_deb_archive( + source_dir, target_dir, output_file_path, build_variables, arch +): + command = _get_command(arch) + subprocess.check_call(command, cwd=source_dir) + deb_arch = _DEB_ARCH[arch] + deb_file_name = f"{build_variables['DEB_PKG_NAME']}_{build_variables['DEB_PKG_VERSION']}_{deb_arch}.deb" + deb_file_path = mozpath.join(target_dir, deb_file_name) + + if not os.path.exists(deb_file_path): + raise NoDebPackageFound(deb_file_path) + + subprocess.check_call(["dpkg-deb", "--info", deb_file_path]) + shutil.move(deb_file_path, output_file_path) + + +def _get_command(arch): + deb_arch = _DEB_ARCH[arch] + command = [ + "dpkg-buildpackage", + # TODO: Use long options once we stop supporting Debian Jesse. They're more + # explicit. + # + # Long options were added in dpkg 1.18.8 which is part of Debian Stretch. + # + # https://git.dpkg.org/cgit/dpkg/dpkg.git/commit/?h=1.18.x&id=293bd243a19149165fc4fd8830b16a51d471a5e9 + # https://packages.debian.org/stretch/dpkg-dev + "-us", # --unsigned-source + "-uc", # --unsigned-changes + "-b", # --build=binary + ] + + if deb_arch != "all": + command.append(f"--host-arch={deb_arch}") + + if _is_chroot_available(arch): + flattened_command = " ".join(command) + command = [ + "chroot", + _get_chroot_path(arch), + "bash", + "-c", + f"cd /tmp/*/source; {flattened_command}", + ] + + return command + + +def _create_temporary_directory(arch): + if _is_chroot_available(arch): + return tempfile.mkdtemp(dir=f"{_get_chroot_path(arch)}/tmp") + else: + return tempfile.mkdtemp() + + +def _is_chroot_available(arch): + return os.path.isdir(_get_chroot_path(arch)) + + +def _get_chroot_path(arch): + deb_arch = "amd64" if arch == "all" else _DEB_ARCH[arch] + return f"/srv/{_DEB_DIST}-{deb_arch}" + + +_MANIFEST_FILE_NAME = "manifest.json" + + +def _extract_langpack_metadata(input_xpi_file): + with tempfile.TemporaryDirectory() as d: + with zipfile.ZipFile(input_xpi_file) as zip: + zip.extract(_MANIFEST_FILE_NAME, path=d) + + with open(mozpath.join(d, _MANIFEST_FILE_NAME)) as f: + return json.load(f) |