From 6bf0a5cb5034a7e684dcc3500e841785237ce2dd Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 19:32:43 +0200 Subject: Adding upstream version 1:115.7.0. Signed-off-by: Daniel Baumann --- comm/taskcluster/docker/tb-atn/Dockerfile | 15 ++ comm/taskcluster/docker/tb-atn/atn_langpack.py | 171 ++++++++++++++++++ comm/taskcluster/docker/tb-atn/make_venv.sh | 12 ++ comm/taskcluster/docker/tb-atn/requirements.in | 3 + comm/taskcluster/docker/tb-atn/requirements.txt | 107 +++++++++++ comm/taskcluster/docker/tb-atn/runme.sh | 12 ++ comm/taskcluster/docker/tb-debian-mingw/Dockerfile | 19 ++ comm/taskcluster/docker/tb-flatpak/Dockerfile | 18 ++ .../docker/tb-flatpak/build_desktop_file.py | 159 +++++++++++++++++ comm/taskcluster/docker/tb-flatpak/close_range.c | 12 ++ .../taskcluster/docker/tb-flatpak/distribution.ini | 13 ++ .../tb-flatpak/extract_locales_from_l10n_json.py | 18 ++ .../docker/tb-flatpak/fluent_requirements.txt | 2 + .../taskcluster/docker/tb-flatpak/launch_script.sh | 3 + comm/taskcluster/docker/tb-flatpak/make_venv.sh | 12 ++ .../org.mozilla.Thunderbird.appdata.xml.in | 83 +++++++++ .../org.mozilla.Thunderbird.desktop.jinja2 | 33 ++++ comm/taskcluster/docker/tb-flatpak/runme.sh | 198 +++++++++++++++++++++ 18 files changed, 890 insertions(+) create mode 100644 comm/taskcluster/docker/tb-atn/Dockerfile create mode 100644 comm/taskcluster/docker/tb-atn/atn_langpack.py create mode 100755 comm/taskcluster/docker/tb-atn/make_venv.sh create mode 100644 comm/taskcluster/docker/tb-atn/requirements.in create mode 100644 comm/taskcluster/docker/tb-atn/requirements.txt create mode 100755 comm/taskcluster/docker/tb-atn/runme.sh create mode 100644 comm/taskcluster/docker/tb-debian-mingw/Dockerfile create mode 100644 comm/taskcluster/docker/tb-flatpak/Dockerfile create mode 100644 comm/taskcluster/docker/tb-flatpak/build_desktop_file.py create mode 100644 comm/taskcluster/docker/tb-flatpak/close_range.c create mode 100644 comm/taskcluster/docker/tb-flatpak/distribution.ini create mode 100644 comm/taskcluster/docker/tb-flatpak/extract_locales_from_l10n_json.py create mode 100644 comm/taskcluster/docker/tb-flatpak/fluent_requirements.txt create mode 100755 comm/taskcluster/docker/tb-flatpak/launch_script.sh create mode 100755 comm/taskcluster/docker/tb-flatpak/make_venv.sh create mode 100644 comm/taskcluster/docker/tb-flatpak/org.mozilla.Thunderbird.appdata.xml.in create mode 100644 comm/taskcluster/docker/tb-flatpak/org.mozilla.Thunderbird.desktop.jinja2 create mode 100755 comm/taskcluster/docker/tb-flatpak/runme.sh (limited to 'comm/taskcluster/docker') diff --git a/comm/taskcluster/docker/tb-atn/Dockerfile b/comm/taskcluster/docker/tb-atn/Dockerfile new file mode 100644 index 0000000000..dcf66f1f83 --- /dev/null +++ b/comm/taskcluster/docker/tb-atn/Dockerfile @@ -0,0 +1,15 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Thunderbird Releng + +VOLUME /builds/worker/workspace + +COPY atn_langpack.py /builds/worker/bin/atn_langpack.py +COPY make_venv.sh /builds/worker/bin/make_venv.sh +COPY runme.sh /builds/worker/bin/runme.sh +COPY requirements.txt /builds/worker/requirements.txt + +RUN /builds/worker/bin/make_venv.sh + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] + diff --git a/comm/taskcluster/docker/tb-atn/atn_langpack.py b/comm/taskcluster/docker/tb-atn/atn_langpack.py new file mode 100644 index 0000000000..fc66c09b2f --- /dev/null +++ b/comm/taskcluster/docker/tb-atn/atn_langpack.py @@ -0,0 +1,171 @@ +#!python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import datetime +import json +import logging +import os +import random +import string +import sys +from enum import Enum +from pathlib import Path +from pprint import pprint as pp +from typing import List, Literal, NamedTuple, Tuple, Union + +import jwt +import requests +from redo import retry + +logging.getLogger("requests").setLevel(logging.DEBUG) + +ATN_UPLOAD_URL = "https://addons.thunderbird.net/api/v3/addons/langpack-{langcode}@thunderbird.mozilla.org/versions/{version}/" +CHUNK_SIZE = 128 * 1024 + + +class ATNChannel(Enum): + LISTED = "listed" + UNLISTED = "unlisted" + + +Locales = List[str] +Version = str +ApiParam = str +EnvVars = NamedTuple( + "EnvVars", + [ + ("langpack_version", Version), + ("locales", Locales), + ("langpack_dir", Path), + ("langpack_channel", Literal[ATNChannel.LISTED, ATNChannel.UNLISTED]), + ("api_key", ApiParam), + ("api_secret", ApiParam), + ], +) +Result = Tuple[str, Union[object, None]] + + +def print_line(message): + msg_bytes = message.encode("utf-8") + written = 0 + while written < len(msg_bytes): + written += sys.stdout.buffer.write(msg_bytes[written:]) or 0 + sys.stdout.buffer.flush() + + +class ATNUploader: + def __init__(self, options: EnvVars): + self.api_key = options.api_key + self.api_secret = options.api_secret + self.langpack_dir = options.langpack_dir + self.langpack_version = options.langpack_version + self.langpack_channel = options.langpack_channel + self.locales = options.locales + + def mk_headers(self) -> dict: + now = datetime.datetime.utcnow() + payload = { + "iss": self.api_key, + "jti": "".join( + random.choice(string.ascii_uppercase + string.digits) for _ in range(64) + ), + "exp": now + datetime.timedelta(seconds=60), + "iat": now, + } + headers = { + "Authorization": "JWT {0}".format( + jwt.encode(payload, self.api_secret, algorithm="HS256") + ) + } + return headers + + def upload_langpack(self, locale: str) -> Result: + langpack_path = self.langpack_dir / locale / "target.langpack.xpi" + headers = self.mk_headers() + langpack_fd = open(langpack_path, "rb") + file = {"upload": ("upload", langpack_fd)} + data = {"channel": self.langpack_channel} + + url = ATN_UPLOAD_URL.format(version=self.langpack_version, langcode=locale) + with requests.put(url, files=file, data=data, headers=headers, verify=False) as resp: + if not resp.ok: + print_line(f"Failed {locale}") + return resp.json() + else: + return resp.json() + + def upload_all_locales(self) -> Tuple[List[Result], List[Result]]: + failed = [] + success = [] + for locale in self.locales: + try: + rv = retry(self.upload_langpack, args=(locale,), attempts=3, sleeptime=10) + if "error" not in rv: + success.append((locale, rv)) + else: + failed.append((locale, rv)) + except requests.HTTPError as e: + print_line(e) + failed.append((locale, None)) + return success, failed + + +def get_secret(name: str) -> Tuple[ApiParam, ApiParam]: + secret = {} + if "MOZ_SCM_LEVEL" in os.environ: + level = os.environ.get("MOZ_SCM_LEVEL", "1") + taskcluster_url = os.environ.get("TASKCLUSTER_PROXY_URL") or os.environ.get( + "TASKCLUSTER_ROOT_URL", "" + ) + secrets_url = ( + f"{taskcluster_url}/secrets/v1/secret/project/comm/thunderbird/releng" + f"/build/level-{level}/{name}" + ) + res = requests.get(secrets_url) + res.raise_for_status() + secret = res.json() + elif "SECRET_FILE" in os.environ: # For local dev/debug + with open(os.environ["SECRET_FILE"]) as fp: + secret = json.load(fp)["secret"] + secret = secret.get("secret") + api_key = secret["api_key"] if "api_key" in secret else None + api_secret = secret["api_secret"] if "api_secret" in secret else None + if api_key is None or api_secret is None: + raise Exception(f"Unable to get secret. {secret.keys()}") + + return api_key, api_secret + + +def read_env_vars() -> EnvVars: + try: + langpack_version = os.environ["LANGPACK_VERSION"] + locales_json = os.environ["LOCALES"] + langpack_dir = Path(os.environ["MOZ_FETCHES_DIR"]).resolve() + langpack_channel = os.environ["ATN_CHANNEL"] + except KeyError: + raise Exception("Missing environment variable(s)") + + locales = json.loads(locales_json) + api_key, api_secret = get_secret("atn_langpack") + + return EnvVars( + langpack_version, locales, langpack_dir, ATNChannel(langpack_channel), api_key, api_secret + ) + + +def main(): + options = read_env_vars() + + atn_uploader = ATNUploader(options) + success, failed = atn_uploader.upload_all_locales() + + pp(success) + if failed: + pp(failed) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/comm/taskcluster/docker/tb-atn/make_venv.sh b/comm/taskcluster/docker/tb-atn/make_venv.sh new file mode 100755 index 0000000000..1ae7b1f881 --- /dev/null +++ b/comm/taskcluster/docker/tb-atn/make_venv.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +set -xe + +cd /builds/worker || exit 1 + +python3 -m venv --system-site-packages venv +source ./venv/bin/activate +python3 -m pip install -r /builds/worker/requirements.txt diff --git a/comm/taskcluster/docker/tb-atn/requirements.in b/comm/taskcluster/docker/tb-atn/requirements.in new file mode 100644 index 0000000000..d6bcfa8d38 --- /dev/null +++ b/comm/taskcluster/docker/tb-atn/requirements.in @@ -0,0 +1,3 @@ +PyJWT==2.7.0 +requests==2.30.0 +redo==2.0.4 diff --git a/comm/taskcluster/docker/tb-atn/requirements.txt b/comm/taskcluster/docker/tb-atn/requirements.txt new file mode 100644 index 0000000000..d223006e6c --- /dev/null +++ b/comm/taskcluster/docker/tb-atn/requirements.txt @@ -0,0 +1,107 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --generate-hashes --output-file=requirements.txt - +# +certifi==2023.7.22 \ + --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 \ + --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9 + # via requests +charset-normalizer==3.2.0 \ + --hash=sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96 \ + --hash=sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c \ + --hash=sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710 \ + --hash=sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706 \ + --hash=sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020 \ + --hash=sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252 \ + --hash=sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad \ + --hash=sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329 \ + --hash=sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a \ + --hash=sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f \ + --hash=sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6 \ + --hash=sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4 \ + --hash=sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a \ + --hash=sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46 \ + --hash=sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2 \ + --hash=sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23 \ + --hash=sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace \ + --hash=sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd \ + --hash=sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982 \ + --hash=sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10 \ + --hash=sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2 \ + --hash=sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea \ + --hash=sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09 \ + --hash=sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5 \ + --hash=sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149 \ + --hash=sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489 \ + --hash=sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9 \ + --hash=sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80 \ + --hash=sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592 \ + --hash=sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3 \ + --hash=sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6 \ + --hash=sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed \ + --hash=sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c \ + --hash=sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200 \ + --hash=sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a \ + --hash=sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e \ + --hash=sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d \ + --hash=sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6 \ + --hash=sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623 \ + --hash=sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669 \ + --hash=sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3 \ + --hash=sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa \ + --hash=sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9 \ + --hash=sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2 \ + --hash=sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f \ + --hash=sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1 \ + --hash=sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4 \ + --hash=sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a \ + --hash=sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8 \ + --hash=sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3 \ + --hash=sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029 \ + --hash=sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f \ + --hash=sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959 \ + --hash=sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22 \ + --hash=sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7 \ + --hash=sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952 \ + --hash=sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346 \ + --hash=sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e \ + --hash=sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d \ + --hash=sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299 \ + --hash=sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd \ + --hash=sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a \ + --hash=sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3 \ + --hash=sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037 \ + --hash=sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94 \ + --hash=sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c \ + --hash=sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858 \ + --hash=sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a \ + --hash=sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449 \ + --hash=sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c \ + --hash=sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918 \ + --hash=sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1 \ + --hash=sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c \ + --hash=sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac \ + --hash=sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa + # via requests +idna==3.4 \ + --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \ + --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2 + # via requests +pyjwt==2.7.0 \ + --hash=sha256:ba2b425b15ad5ef12f200dc67dd56af4e26de2331f965c5439994dad075876e1 \ + --hash=sha256:bd6ca4a3c4285c1a2d4349e5a035fdf8fb94e04ccd0fcbe6ba289dae9cc3e074 + # via -r - +redo==2.0.4 \ + --hash=sha256:81066955041c853b0e6491eb65a0877dce45131c4cfa3d42d923fc2aa8f7a043 \ + --hash=sha256:c76e4c23ab2f8840261736a851323cd98493710e7a9d36a1058535dca501f293 + # via -r - +requests==2.30.0 \ + --hash=sha256:10e94cc4f3121ee6da529d358cdaeaff2f1c409cd377dbc72b825852f2f7e294 \ + --hash=sha256:239d7d4458afcb28a692cdd298d87542235f4ca8d36d03a15bfc128a6559a2f4 + # via -r - +urllib3==2.0.4 \ + --hash=sha256:8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11 \ + --hash=sha256:de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4 + # via requests diff --git a/comm/taskcluster/docker/tb-atn/runme.sh b/comm/taskcluster/docker/tb-atn/runme.sh new file mode 100755 index 0000000000..bf504d3735 --- /dev/null +++ b/comm/taskcluster/docker/tb-atn/runme.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +set -xe + +cd /builds/worker || exit 1 + +source venv/bin/activate + +exec python3 bin/atn_langpack.py diff --git a/comm/taskcluster/docker/tb-debian-mingw/Dockerfile b/comm/taskcluster/docker/tb-debian-mingw/Dockerfile new file mode 100644 index 0000000000..ab5a7dbc2c --- /dev/null +++ b/comm/taskcluster/docker/tb-debian-mingw/Dockerfile @@ -0,0 +1,19 @@ +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Rob Lemley +# Used by Thunderbird to build third party libraries for OTR messaging. + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache + +RUN apt-get update && \ + apt-get dist-upgrade && \ + apt-get install \ + autoconf \ + automake \ + binutils-mingw-w64 \ + gcc-mingw-w64 \ + gcc-mingw-w64-i686 \ + gcc-mingw-w64-x86-64 \ + libtool \ + mingw-w64-tools diff --git a/comm/taskcluster/docker/tb-flatpak/Dockerfile b/comm/taskcluster/docker/tb-flatpak/Dockerfile new file mode 100644 index 0000000000..0ec129c9f3 --- /dev/null +++ b/comm/taskcluster/docker/tb-flatpak/Dockerfile @@ -0,0 +1,18 @@ +FROM freedesktopsdk/flatpak:22.08-x86_64 +MAINTAINER tb-builds@thunderbird.net + +RUN mkdir /scripts/ +WORKDIR /scripts/ + +# Copy everything in the docker/tb-flatpak folder but the Dockerfile +COPY [^D]* /scripts/ + +# Set up Python virtual environment +RUN /scripts/make_venv.sh + +# Manually add close_range syscall to image +RUN ["gcc", "-Wall", "-shared", "-o", "/scripts/close_range.so", "/scripts/close_range.c"] +ENV LD_PRELOAD /scripts/close_range.so + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/comm/taskcluster/docker/tb-flatpak/build_desktop_file.py b/comm/taskcluster/docker/tb-flatpak/build_desktop_file.py new file mode 100644 index 0000000000..65ed76eeb8 --- /dev/null +++ b/comm/taskcluster/docker/tb-flatpak/build_desktop_file.py @@ -0,0 +1,159 @@ +#!/usr/bin/python3 -u +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +""" +Build the Flatpak .desktop file. Needs to run in the Python virtualenv +due to dependencies. + +python3 /scripts/build_desktop_file.py -o "$WORKSPACE/org.mozilla.Thunderbird.desktop" \ + -t "/scripts/org.mozilla.Thunderbird.desktop.jinja2" \ + -l "$WORKSPACE/l10n-central" \ + -L "$WORKSPACE/shipped-locales" \ + -f "mail/branding/thunderbird/brand.ftl" \ + -f "mail/messenger/flatpak.ftl" +""" + +import argparse +import json +import os +import urllib.request +import zipfile +from pathlib import Path +from typing import List, Union + +import jinja2 +from fluent.runtime.fallback import FluentLocalization, FluentResourceLoader + +COMM_L10N_ZIP = "https://hg.mozilla.org/projects/comm-l10n/archive/{rev}.zip" +COMM_L10N_ZIP_PREFIX = "comm-l10n-{rev}" + + +class FluentTranslator: + """ + FluentTranslator is an enhanced FluentLocalization + """ + + def __init__(self, l10n_base: Path, locales: List[str], resource_ids: List[str]): + self._locales = locales + self._localizations = self._populate(l10n_base, resource_ids) + + @property + def locales(self): + return sorted([l for l in self._locales if l != "en-US"]) + + def _populate(self, l10n_path, resource_ids): + loader = FluentResourceLoader(str(l10n_path / "{locale}")) + + rv = {} + for locale in self._locales: + rv[locale] = FluentLocalization([locale], resource_ids, loader) + + return rv + + def get_message(self, locale, message_id) -> Union[str, None]: + rv = self._localizations[locale].format_value(message_id) + if rv == message_id: + return None + return rv + + +def get_multi_translate(l10n_strings: FluentTranslator): + def translate_multi(key: str, fluent_id: str): + for locale in l10n_strings.locales: + translated = l10n_strings.get_message(locale, fluent_id) + if translated is not None: + yield f"{key}[{locale}]={translated}" + + return translate_multi + + +def build_template( + output: Path, + template: Path, + l10n_base: Path, + locales: List[str], + fluent_resources: List[str], + is_beta: bool, +): + wmclass = "thunderbird" + if is_beta: + wmclass = wmclass + "-beta" + locales_plus = locales + ["en-US"] + l10n_strings = FluentTranslator(l10n_base.resolve(), locales_plus, fluent_resources) + + with open(template) as fp: + jinja_template = jinja2.Template(fp.read()) + + translate_multi = get_multi_translate(l10n_strings) + result = jinja_template.render( + strings=l10n_strings, translate=translate_multi, wmclass=wmclass + ) + + with open(output, "w") as fp: + fp.write(result) + + +def get_extract_members( + zip_file: zipfile.ZipFile, file_pats: List[str], prefix: str +) -> List[zipfile.ZipInfo]: + for m in zip_file.infolist(): + for pat in file_pats: + if m.filename.endswith(pat): + m.filename = os.path.relpath(m.filename, prefix) + print(f"Found {m.filename} in strings repo.") + yield m + + +def get_strings(l10n_base, rev, fluent_files): + url = COMM_L10N_ZIP.format(rev=rev) + temp_file, headers = urllib.request.urlretrieve(url) + with zipfile.ZipFile(temp_file, "r") as strings_zip: + to_extract = get_extract_members( + strings_zip, fluent_files, COMM_L10N_ZIP_PREFIX.format(rev=rev) + ) + + strings_zip.extractall(path=l10n_base, members=to_extract) + + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument("-o", dest="output", type=Path, required=True, help="Output file") + parser.add_argument( + "-t", dest="template", type=Path, required=True, help="Jinja2 template file" + ) + parser.add_argument( + "-l", dest="l10n_base", type=Path, required=True, help="l10n-central root path" + ) + parser.add_argument( + "-L", dest="locales_file", type=Path, required=True, help="List of supported locales" + ) + parser.add_argument( + "-f", dest="fluent_files", type=str, required=True, action="extend", nargs="+" + ) + parser.add_argument( + "--beta", + dest="is_beta", + action="store_true", + default=False, + help="Mark this build a beta version", + ) + + args = parser.parse_args() + + with open(args.locales_file) as fp: + locale_data = json.load(fp) + locales = [l for l in locale_data.keys() if l != "ja-JP-mac"] + comm_l10n_rev = locale_data.get("en-GB", {}).get("revision") + + get_strings(args.l10n_base, comm_l10n_rev, args.fluent_files) + + build_template( + args.output, args.template, args.l10n_base, locales, args.fluent_files, args.is_beta + ) + + +if __name__ == "__main__": + main() diff --git a/comm/taskcluster/docker/tb-flatpak/close_range.c b/comm/taskcluster/docker/tb-flatpak/close_range.c new file mode 100644 index 0000000000..d786e78e3b --- /dev/null +++ b/comm/taskcluster/docker/tb-flatpak/close_range.c @@ -0,0 +1,12 @@ +/* + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. +*/ + +#include + +int close_range(unsigned int first, unsigned int last, unsigned int flags) { + errno = ENOSYS; + return -1; +} diff --git a/comm/taskcluster/docker/tb-flatpak/distribution.ini b/comm/taskcluster/docker/tb-flatpak/distribution.ini new file mode 100644 index 0000000000..d7793363a1 --- /dev/null +++ b/comm/taskcluster/docker/tb-flatpak/distribution.ini @@ -0,0 +1,13 @@ +[Global] +id=thunderbird-flatpak +version=1.0 +about=Mozilla Thunderbird Flatpak +about.en-US=Mozilla Thunderbird Flatpak en-US + +[Preferences] +intl.locale.requested="" +app.update.auto=false +app.update.enabled=false +app.update.autoInstallEnabled=false +mail.shell.checkDefaultClient=false +spellchecker.dictionary_path=/usr/share/hunspell diff --git a/comm/taskcluster/docker/tb-flatpak/extract_locales_from_l10n_json.py b/comm/taskcluster/docker/tb-flatpak/extract_locales_from_l10n_json.py new file mode 100644 index 0000000000..b1eb745d7f --- /dev/null +++ b/comm/taskcluster/docker/tb-flatpak/extract_locales_from_l10n_json.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +""" +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +""" + + +import json +import sys + +l10n_changesets_json_path = sys.argv[1] +with open(l10n_changesets_json_path) as f: + locales = json.load(f).keys() +linux_locales = [l for l in locales if l != "ja-JP-mac"] + +print("\n".join(sorted(linux_locales))) diff --git a/comm/taskcluster/docker/tb-flatpak/fluent_requirements.txt b/comm/taskcluster/docker/tb-flatpak/fluent_requirements.txt new file mode 100644 index 0000000000..4b9fde4a30 --- /dev/null +++ b/comm/taskcluster/docker/tb-flatpak/fluent_requirements.txt @@ -0,0 +1,2 @@ +fluent.runtime==0.4.0 +jinja2==3.1.2 diff --git a/comm/taskcluster/docker/tb-flatpak/launch_script.sh b/comm/taskcluster/docker/tb-flatpak/launch_script.sh new file mode 100755 index 0000000000..27875568b9 --- /dev/null +++ b/comm/taskcluster/docker/tb-flatpak/launch_script.sh @@ -0,0 +1,3 @@ +#!/bin/bash +export TMPDIR="$XDG_RUNTIME_DIR/app/$FLATPAK_ID" +exec /app/lib/thunderbird/thunderbird --name org.mozilla.Thunderbird "$@" diff --git a/comm/taskcluster/docker/tb-flatpak/make_venv.sh b/comm/taskcluster/docker/tb-flatpak/make_venv.sh new file mode 100755 index 0000000000..6b807e64f4 --- /dev/null +++ b/comm/taskcluster/docker/tb-flatpak/make_venv.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +set -xe + +cd /scripts || exit 1 + +python -m venv --system-site-packages venv +source ./venv/bin/activate +python -m pip install -r fluent_requirements.txt diff --git a/comm/taskcluster/docker/tb-flatpak/org.mozilla.Thunderbird.appdata.xml.in b/comm/taskcluster/docker/tb-flatpak/org.mozilla.Thunderbird.appdata.xml.in new file mode 100644 index 0000000000..5a103650ce --- /dev/null +++ b/comm/taskcluster/docker/tb-flatpak/org.mozilla.Thunderbird.appdata.xml.in @@ -0,0 +1,83 @@ + + + org.mozilla.Thunderbird + org.mozilla.Thunderbird.desktop + CC0-1.0 + Thunderbird + Thunderbird is a free and open source email, newsfeed, chat, and calendaring client + + + +

+ Thunderbird is a free and open source email, newsfeed, chat, and + calendaring client, that’s easy to set up and customize. One of the core + principles of Thunderbird is the use and promotion of open standards - + this focus is a rejection of our world of closed platforms and services + that can’t communicate with each other. We want our users to have freedom + and choice in how they communicate. +

+

+ Thunderbird is an open source project, which means anyone can contribute + ideas, designs, code, and time helping fellow users. +

+ + + + + $RELEASE_NOTES_URL + + + + + mozilla + mail + email + calendar + + + + Calendar + Chat + ContactManagement + Email + Feed + InstantMessaging + IRCClient + Network + News + Office + + + + message/rfc822 + x-scheme-handler/mailto + text/calendar + text/vcard + text/x-vcard + x-scheme-handler/webcal + x-scheme-handler/webcals + x-scheme-handler/mid + + + + https://www.thunderbird.net/ + https://bugzilla.mozilla.org/ + https://support.mozilla.org/kb/thunderbird-faq/ + https://support.mozilla.org/products/thunderbird/ + https://give.thunderbird.net/ + https://www.thunderbird.net/en-US/get-involved/#translation + https://www.thunderbird.net/contact/ + + + https://raw.githubusercontent.com/thunderbird/flatpak-screenshots/main/image_1.png + https://raw.githubusercontent.com/thunderbird/flatpak-screenshots/main/image_2.png + + + + $MANIFEST_URL + + + Mozilla + MPL-2.0 + MZLA Technologies, part of the Mozilla Foundation + diff --git a/comm/taskcluster/docker/tb-flatpak/org.mozilla.Thunderbird.desktop.jinja2 b/comm/taskcluster/docker/tb-flatpak/org.mozilla.Thunderbird.desktop.jinja2 new file mode 100644 index 0000000000..3623c55302 --- /dev/null +++ b/comm/taskcluster/docker/tb-flatpak/org.mozilla.Thunderbird.desktop.jinja2 @@ -0,0 +1,33 @@ +[Desktop Entry] +Name={{ strings.get_message("en-US", "flatpak-desktop-name") }} +Comment={{ strings.get_message("en-US", "flatpak-desktop-comment") }} +{%- for line in translate("Comment", "flatpak-desktop-comment") %} +{{ line }} +{%- endfor %} +GenericName={{ strings.get_message("en-US", "flatpak-desktop-generic-name") }} +{%- for line in translate("GenericName", "flatpak-desktop-generic-name") %} +{{ line }} +{%- endfor %} +Exec=thunderbird %u +Terminal=false +Type=Application +Icon=org.mozilla.Thunderbird +Categories=Network;Email; +MimeType=message/rfc822;x-scheme-handler/mailto;text/calendar;text/vcard;text/x-vcard;x-scheme-handler/webcal;x-scheme-handler/webcals;x-scheme-handler/mid; +StartupNotify=true +StartupWMClass={{ wmclass }} +Actions=ComposeMessage;OpenAddressBook; + +[Desktop Action ComposeMessage] +Name={{ strings.get_message("en-US", "flatpak-desktop-action-compose") }} +{%- for line in translate("Name", "flatpak-desktop-action-compose") %} +{{ line }} +{%- endfor %} +Exec=thunderbird -compose + +[Desktop Action OpenAddressBook] +Name={{ strings.get_message("en-US", "flatpak-desktop-action-addressbook") }} +{%- for line in translate("Name", "flatpak-desktop-action-addressbook") %} +{{ line }} +{%- endfor %} +Exec=thunderbird -addressbook diff --git a/comm/taskcluster/docker/tb-flatpak/runme.sh b/comm/taskcluster/docker/tb-flatpak/runme.sh new file mode 100755 index 0000000000..389929a124 --- /dev/null +++ b/comm/taskcluster/docker/tb-flatpak/runme.sh @@ -0,0 +1,198 @@ +#!/bin/bash +set -xe + +# Future products supporting Flatpaks will set this accordingly +: PRODUCT "${PRODUCT:=thunderbird}" + +# Required environment variables +test "$VERSION" +test "$BUILD_NUMBER" +test "$CANDIDATES_DIR" +test "$L10N_CHANGESETS" +test "$FLATPAK_BRANCH" +test "$MANIFEST_URL" +test "$RELEASE_NOTES_URL" + +# Optional environment variables +: WORKSPACE "${WORKSPACE:=/home/worker/workspace}" +: ARTIFACTS_DIR "${ARTIFACTS_DIR:=/home/worker/artifacts}" + +# Populate remaining environment variables +SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +TARGET_TAR_XZ_FULL_PATH="$ARTIFACTS_DIR/target.flatpak.tar.xz" +SOURCE_DEST="${WORKSPACE}/source" +DISTRIBUTION_DIR="$SOURCE_DEST/distribution" +FREEDESKTOP_VERSION="23.08" +FIREFOX_BASEAPP_CHANNEL="23.08" + +# Create alias for ideal curl command +CURL="curl --location --retry 10 --retry-delay 10" + +# Get current date +# +# This is used to populate the datetime in org.mozilla.Thunderbird.appdata.xml +DATE=$(date +%Y-%m-%d) +export DATE + +# Prepare directories +# +# This command is temporary, there's an upcoming fix in the upstream +# Docker image that we work on top of, from 'freedesktopsdk', that will +# make these two lines go away eventually. +mkdir -p /root /tmp /var/tmp +mkdir -p "$ARTIFACTS_DIR" +rm -rf "$SOURCE_DEST" && mkdir -p "$SOURCE_DEST" + +# Ensure a clean slate in the local Flatpak repo +rm -rf ~/.local/share/flatpak/ + +# Download en-US linux64 (English, 64-bit Linux) Thunderbird binary +$CURL -o "${WORKSPACE}/thunderbird.tar.bz2" \ + "${CANDIDATES_DIR}/${VERSION}-candidates/build${BUILD_NUMBER}/linux-x86_64/en-US/thunderbird-${VERSION}.tar.bz2" + +# Fetch list of Thunderbird locales +$CURL -o "${WORKSPACE}/l10n-changesets.json" "$L10N_CHANGESETS" +locales=$(python3 "$SCRIPT_DIRECTORY/extract_locales_from_l10n_json.py" "${WORKSPACE}/l10n-changesets.json") + +# Fetch langpack extension for each locale +mkdir -p "$DISTRIBUTION_DIR" +mkdir -p "$DISTRIBUTION_DIR/extensions" +for locale in $locales; do + $CURL -o "$DISTRIBUTION_DIR/extensions/langpack-${locale}@thunderbird.mozilla.org.xpi" \ + "$CANDIDATES_DIR/${VERSION}-candidates/build${BUILD_NUMBER}/linux-x86_64/xpi/${locale}.xpi" +done + +# Download artifacts from dependencies and build the .desktop file. +( +source /scripts/venv/bin/activate +python3 /scripts/build_desktop_file.py -o "$WORKSPACE/org.mozilla.Thunderbird.desktop" \ + -t "/scripts/org.mozilla.Thunderbird.desktop.jinja2" \ + -l "$WORKSPACE/l10n-central" \ + -L "$WORKSPACE/l10n-changesets.json" \ + -f "mail/branding/thunderbird/brand.ftl" \ + -f "mail/messenger/flatpak.ftl" +) + +# Generate AppData XML from template, add various +envsubst < "$SCRIPT_DIRECTORY/org.mozilla.Thunderbird.appdata.xml.in" > "${WORKSPACE}/org.mozilla.Thunderbird.appdata.xml" +cp -v "$SCRIPT_DIRECTORY/distribution.ini" "$WORKSPACE" +cp -v "$SCRIPT_DIRECTORY/launch_script.sh" "$WORKSPACE" +cd "${WORKSPACE}" + +# Fetch and install Firefox base app (as user, not system-wide) +flatpak remote-add --user --if-not-exists --from flathub https://dl.flathub.org/repo/flathub.flatpakrepo +flatpak install --user -y flathub org.mozilla.firefox.BaseApp//${FIREFOX_BASEAPP_CHANNEL} --no-deps + +# Create build directory and add Firefox base app files +# +# This command is temporary, there's an upcoming fix in the upstream +# Docker image that we work on top of, from 'freedesktopsdk', that will +# make these two lines go away eventually. +mkdir -p build +cp -r ~/.local/share/flatpak/app/org.mozilla.firefox.BaseApp/current/active/files build/files + +# Create Flatpak build metadata file for Thunderbird +ARCH=$(flatpak --default-arch) +cat < build/metadata +[Application] +name=org.mozilla.Thunderbird +runtime=org.freedesktop.Platform/${ARCH}/${FREEDESKTOP_VERSION} +sdk=org.freedesktop.Sdk/${ARCH}/${FREEDESKTOP_VERSION} +base=app/org.mozilla.firefox.BaseApp/${ARCH}/${FIREFOX_BASEAPP_CHANNEL} +[Extension org.mozilla.Thunderbird.Locale] +directory=share/runtime/langpack +autodelete=true +locale-subset=true +EOF + +# Create Flatpak build metadata file for locales +cat < build/metadata.locale +[Runtime] +name=org.mozilla.Thunderbird.Locale + +[ExtensionOf] +ref=app/org.mozilla.Thunderbird/${ARCH}/${FLATPAK_BRANCH} +EOF + +# Install Thunderbird files into appdir +appdir=build/files +install -d "${appdir}/lib/" +(cd "${appdir}/lib/" && tar jxf "${WORKSPACE}/thunderbird.tar.bz2") +install -D -m644 -t "${appdir}/share/appdata" org.mozilla.Thunderbird.appdata.xml +install -D -m644 -t "${appdir}/share/applications" org.mozilla.Thunderbird.desktop +for size in 16 32 48 64 128; do + install -D -m644 "${appdir}/lib/thunderbird/chrome/icons/default/default${size}.png" "${appdir}/share/icons/hicolor/${size}x${size}/apps/org.mozilla.Thunderbird.png" +done + +# Generate AppStream metadata and add screenshots from Flathub +appstream-compose --prefix="${appdir}" --origin=flatpak --basename=org.mozilla.Thunderbird org.mozilla.Thunderbird +appstream-util mirror-screenshots "${appdir}"/share/app-info/xmls/org.mozilla.Thunderbird.xml.gz "https://dl.flathub.org/repo/screenshots/org.mozilla.Thunderbird-${FLATPAK_BRANCH}" build/screenshots "build/screenshots/org.mozilla.Thunderbird-${FLATPAK_BRANCH}" + +# Install locales, distribution, and launch_script.sh into appdir +# +# We must install each locale individually, since we're symlinking +# each one. +# +# We put the langpacks in /app/share/locale/$LANG_CODE and symlink that +# directory to where Thunderbird looks them up; this way only the subset +# of locales configured on the user's system are downloaded, instead of +# all locales. +mkdir -p "${appdir}/lib/thunderbird/distribution/extensions" +for locale in $locales; do + install -D -m644 -t "${appdir}/share/runtime/langpack/${locale%%-*}/" "${DISTRIBUTION_DIR}/extensions/langpack-${locale}@thunderbird.mozilla.org.xpi" + ln -sf "/app/share/runtime/langpack/${locale%%-*}/langpack-${locale}@thunderbird.mozilla.org.xpi" "${appdir}/lib/thunderbird/distribution/extensions/langpack-${locale}@thunderbird.mozilla.org.xpi" +done +install -D -m644 -t "${appdir}/lib/thunderbird/distribution" distribution.ini +install -D -m755 launch_script.sh "${appdir}/bin/thunderbird" + +# Build Flatpak +# +# We use features=devel to enable ptrace, which we need for the crash +# reporter. The application is still confined in a pid namespace, so +# that won't let us escape the flatpak sandbox. See bug 1653852. +# +# We use own-name to ensure Thunderbird has access to DBus, as app ID +# (org.mozilla.Thunderbird) does not match bus names +# (org.mozilla.thunderbird, lowercase "t"). The app ID may be updated +# in the future to match the default bus names. +flatpak build-finish build \ + --allow=devel \ + --share=ipc \ + --share=network \ + --socket=pulseaudio \ + --socket=wayland \ + --socket=x11 \ + --socket=pcsc \ + --socket=cups \ + --require-version=0.10.3 \ + --persist=.thunderbird \ + --filesystem=xdg-download:rw \ + --filesystem=~/.gnupg \ + --filesystem=xdg-run/gnupg:ro \ + --filesystem=xdg-run/speech-dispatcher:ro \ + --filesystem=/run/.heim_org.h5l.kcm-socket \ + --device=dri \ + --own-name="org.mozilla.thunderbird.*" \ + --own-name="org.mozilla.thunderbird_beta.*" \ + --talk-name="org.gtk.vfs.*" \ + --talk-name=org.a11y.Bus \ + --system-talk-name=org.freedesktop.NetworkManager \ + --command=thunderbird + +# Export Flatpak build into repo +flatpak build-export --disable-sandbox --no-update-summary --exclude='/share/runtime/langpack/*/*' repo build "$FLATPAK_BRANCH" +flatpak build-export --disable-sandbox --no-update-summary --metadata=metadata.locale --files=files/share/runtime/langpack repo build "$FLATPAK_BRANCH" + +# Commit screenshots to repo +ostree commit --repo=repo --canonical-permissions --branch=screenshots/x86_64 build/screenshots +flatpak build-update-repo --generate-static-deltas repo + +# Package Flatpak repo as tar +tar cvfJ flatpak.tar.xz repo +mv -- flatpak.tar.xz "$TARGET_TAR_XZ_FULL_PATH" + +# Build Flatpak bundle (.flatpak) from repo +flatpak build-bundle "$WORKSPACE"/repo org.mozilla.Thunderbird.flatpak org.mozilla.Thunderbird "$FLATPAK_BRANCH" + +# Move bundle to artifacts +mv org.mozilla.Thunderbird.flatpak "$ARTIFACTS_DIR/" -- cgit v1.2.3