From 6bf0a5cb5034a7e684dcc3500e841785237ce2dd Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 19:32:43 +0200 Subject: Adding upstream version 1:115.7.0. Signed-off-by: Daniel Baumann --- comm/taskcluster/scripts/are-we-esmified-yet.py | 147 +++++++++ comm/taskcluster/scripts/build-l10n-pre.sh | 90 ++++++ comm/taskcluster/scripts/build-libotr.sh | 360 +++++++++++++++++++++ comm/taskcluster/scripts/build-source-docs.sh | 34 ++ comm/taskcluster/scripts/desktop_comm_l10n.py | 231 +++++++++++++ .../scripts/source-test-clang-format.sh | 32 ++ 6 files changed, 894 insertions(+) create mode 100644 comm/taskcluster/scripts/are-we-esmified-yet.py create mode 100755 comm/taskcluster/scripts/build-l10n-pre.sh create mode 100755 comm/taskcluster/scripts/build-libotr.sh create mode 100755 comm/taskcluster/scripts/build-source-docs.sh create mode 100755 comm/taskcluster/scripts/desktop_comm_l10n.py create mode 100755 comm/taskcluster/scripts/source-test-clang-format.sh (limited to 'comm/taskcluster/scripts') diff --git a/comm/taskcluster/scripts/are-we-esmified-yet.py b/comm/taskcluster/scripts/are-we-esmified-yet.py new file mode 100644 index 0000000000..20eaf108fd --- /dev/null +++ b/comm/taskcluster/scripts/are-we-esmified-yet.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import pathlib +import subprocess +import sys + +TBPL_FAILURE = 2 + +excluded_prefix = [ + "suite/", +] +EXCLUSION_FILES = [ + os.path.join("tools", "lint", "ThirdPartyPaths.txt"), +] + + +if not (pathlib.Path(".hg").exists() and pathlib.Path("mail/moz.configure").exists()): + print( + "This script needs to be run inside mozilla-central + comm-central " + "checkout of mercurial. " + ) + sys.exit(TBPL_FAILURE) + + +def load_exclusion_files(): + for path in EXCLUSION_FILES: + with open(path, "r") as f: + for line in f: + excluded_prefix.append(line.strip()) + + +def is_excluded(path): + """Returns true if the JSM file shouldn't be converted to ESM.""" + path_str = str(path) + for prefix in excluded_prefix: + if path_str.startswith(prefix): + return True + + return False + + +def new_files_struct(): + return { + "jsm": [], + "esm": [], + "subdir": {}, + } + + +def put_file(files, kind, path): + """Put a path into files tree structure.""" + + if is_excluded(path): + return + + name = path.name + + current_files = files + for part in path.parent.parts: + if part not in current_files["subdir"]: + current_files["subdir"][part] = new_files_struct() + current_files = current_files["subdir"][part] + + current_files[kind].append(name) + + +def run(cmd): + """Run command and return output as lines, excluding empty line.""" + lines = subprocess.run(cmd, stdout=subprocess.PIPE).stdout.decode() + return filter(lambda x: x != "", lines.split("\n")) + + +def collect_jsm(files): + """Collect JSM files.""" + kind = "jsm" + + # jsm files + cmd = ["hg", "files", "set:glob:**/*.jsm"] + for line in run(cmd): + put_file(files, kind, pathlib.Path(line)) + + # js files with EXPORTED_SYMBOLS + cmd = ["hg", "files", "set:grep('EXPORTED_SYMBOLS = \[') and glob:**/*.js"] + for line in run(cmd): + put_file(files, kind, pathlib.Path(line)) + + +def collect_esm(files): + """Collect system ESM files.""" + kind = "esm" + + # sys.mjs files + cmd = ["hg", "files", "set:glob:**/*.sys.mjs"] + + for line in run(cmd): + put_file(files, kind, pathlib.Path(line)) + + +def to_stat(files): + """Convert files tree into status tree.""" + jsm = len(files["jsm"]) + esm = len(files["esm"]) + subdir = {} + + for key, sub_files in files["subdir"].items(): + sub_stat = to_stat(sub_files) + + subdir[key] = sub_stat + jsm += sub_stat["jsm"] + esm += sub_stat["esm"] + + stat = { + "jsm": jsm, + "esm": esm, + } + if len(subdir): + stat["subdir"] = subdir + + return stat + + +def main(): + cmd = ["hg", "parent", "--template", "{node}"] + commit_hash = list(run(cmd))[0] + + cmd = ["hg", "parent", "--template", "{date|shortdate}"] + date = list(run(cmd))[0] + + files = new_files_struct() + collect_jsm(files) + collect_esm(files) + + stat = to_stat(files) + stat["hash"] = commit_hash + stat["date"] = date + + print(json.dumps(stat, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/comm/taskcluster/scripts/build-l10n-pre.sh b/comm/taskcluster/scripts/build-l10n-pre.sh new file mode 100755 index 0000000000..7482dc7c5a --- /dev/null +++ b/comm/taskcluster/scripts/build-l10n-pre.sh @@ -0,0 +1,90 @@ +#! /bin/bash -vex + +set -x -e + +echo "running as" $(id) + +#### +# Taskcluster friendly wrapper for performing fx desktop l10n repacks via mozharness. +# Based on ./build-l10n.sh +#### + +# Inputs, with defaults + +: MOZHARNESS_SCRIPT ${MOZHARNESS_SCRIPT} +: MOZHARNESS_CONFIG ${MOZHARNESS_CONFIG} +: MOZHARNESS_CONFIG_PATHS ${MOZHARNESS_CONFIG_PATHS} +: MOZHARNESS_ACTIONS ${MOZHARNESS_ACTIONS} +: MOZHARNESS_OPTIONS ${MOZHARNESS_OPTIONS} + +: TOOLTOOL_CACHE ${TOOLTOOL_CACHE:=/builds/worker/tooltool-cache} + +: MOZ_SCM_LEVEL ${MOZ_SCM_LEVEL:=1} + +: MOZ_SCM_LEVEL ${MOZ_SCM_LEVEL:=1} + +: WORKSPACE ${WORKSPACE:=/builds/worker/workspace} +: MOZ_OBJDIR ${MOZ_OBJDIR:=$WORKSPACE/obj-build} + +set -v + +fail() { + echo # make sure error message is on a new line + echo "[build-l10n-pre.sh:error]" "${@}" + exit 1 +} + +export MOZ_CRASHREPORTER_NO_REPORT=1 +export TINDERBOX_OUTPUT=1 + +# test required parameters are supplied +if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi +if [[ -z "${MOZHARNESS_CONFIG}" && -z "${EXTRA_MOZHARNESS_CONFIG}" ]]; then fail "MOZHARNESS_CONFIG or EXTRA_MOZHARNESS_CONFIG is not set"; fi + +# set up mozharness configuration, via command line, env, etc. + +# $TOOLTOOL_CACHE bypasses mozharness completely and is read by tooltool_wrapper.sh to set the +# cache. However, only some mozharness scripts use tooltool_wrapper.sh, so this may not be +# entirely effective. +export TOOLTOOL_CACHE + +export MOZ_OBJDIR + +config_path_cmds="" +for path in ${MOZHARNESS_CONFIG_PATHS}; do + config_path_cmds="${config_path_cmds} --extra-config-path ${GECKO_PATH}/${path}" +done + +# support multiple, space delimited, config files +config_cmds="" +for cfg in $MOZHARNESS_CONFIG; do + config_cmds="${config_cmds} --config ${cfg}" +done + +# if MOZHARNESS_ACTIONS is given, only run those actions (completely overriding default_actions +# in the mozharness configuration) +if [ -n "$MOZHARNESS_ACTIONS" ]; then + actions="" + for action in $MOZHARNESS_ACTIONS; do + actions="$actions --$action" + done +fi + +# if MOZHARNESS_OPTIONS is given, append them to mozharness command line run +if [ -n "$MOZHARNESS_OPTIONS" ]; then + options="" + for option in $MOZHARNESS_OPTIONS; do + options="$options --$option" + done +fi + +cd /builds/worker + +$GECKO_PATH/mach python -- \ + $GECKO_PATH/${MOZHARNESS_SCRIPT} \ + ${config_path_cmds} \ + ${config_cmds} \ + $actions \ + $options \ + --log-level=debug \ + --work-dir=$WORKSPACE \ diff --git a/comm/taskcluster/scripts/build-libotr.sh b/comm/taskcluster/scripts/build-libotr.sh new file mode 100755 index 0000000000..60df92c531 --- /dev/null +++ b/comm/taskcluster/scripts/build-libotr.sh @@ -0,0 +1,360 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +set -x -eE -v + +_TARGET_OS="$1" + +# Environment variables that are set by Taskcluster. +GECKO_PATH=${GECKO_PATH:-"/builds/worker/workspace/build/src"} +MOZ_FETCHES_DIR=${MOZ_FETCHES_DIR:-"/builds/worker/fetches"} +UPLOAD_DIR=${UPLOAD_DIR:-"/builds/worker/artifacts"} +WORKSPACE=${WORKSPACE:-"${HOME}/workspace"} +MACOS_SDK_DIR=${MACOS_SDK_DIR:-"MacOSX11.3.sdk"} +MACOS_TARGET_SDK=${MACOS_TARGET_SDK:-"10.12"} + + +# Set $DEVEL_TESTING during script development on a local machine +if [[ -n ${DEVEL_TESTING} ]]; then + rm -rf "${UPLOAD_DIR}" "${WORKSPACE}" + mkdir "${UPLOAD_DIR}" "${WORKSPACE}" +fi + +cd "$WORKSPACE" +if [[ ! -d build ]]; then + mkdir build +fi +for _d in build/libgpg-error build/libgcrypt build/libotr build/build_prefix; do + if [[ -e "${_d}" ]]; then + rm -rf "${_d}" + fi +done +BUILD="${WORKSPACE}/build" + +COMPRESS_EXT=xz + +_INSTDIR="build_prefix" +_PREFIX="${BUILD}/${_INSTDIR}" + +_ARTIFACT_STAGEDIR="${BUILD}/libotr_stage" + +THIRD_PARTY_SRC="${GECKO_PATH}/comm/third_party" + +GPG_ERROR_SRC="${BUILD}/libgpg-error" +GCRYPT_SRC="${BUILD}/libgcrypt" +OTR_SRC="${BUILD}/libotr" + +# Set environment variables needed for all dependencies +_BASE_CONFIGURE=(--build=x86_64-pc-linux --prefix="${_PREFIX}" --disable-silent-rules) + +_OS_CONFIGURE_FLAGS=() # May be overridden per target OS + + +function clang_cfg() { + # autotools and friends seem to work better with Clang if the compiler + # is named -clang. This applies to macOS only. It does not seem + # necessary when building for Linux. + local _i _clang_cfg_dir _clang_dir + + _clang_cfg_dir="${THIRD_PARTY_SRC}/clang" + _clang_dir="${MOZ_FETCHES_DIR}/clang/bin" + + cp -a "${_clang_cfg_dir}"/*.cfg "${_clang_dir}" + for _i in x86_64-apple-darwin aarch64-apple-darwin aarch64-linux-gnu i686-linux-gnu; do + ln -s clang "${_clang_dir}/${_i}-clang" + done + return 0 +} + +function copy_sources() { + # The checkout directory should be treated readonly + local _pkg + cd "${BUILD}" + for _pkg in libgpg-error libgcrypt libotr; do + cp -a "${THIRD_PARTY_SRC}/${_pkg}" . + done +} + +function build_libgpg-error() { + echo "Building libgpg-error" + cd "${GPG_ERROR_SRC}" + + ./configure "${_CONFIGURE_FLAGS[@]}" "${_CONF_STATIC[@]}" \ + --disable-tests --disable-doc --with-pic + + # Hack... *sigh* + if [[ "${_TARGET_OS}" == "linux-aarch64" ]]; then + cp src/syscfg/lock-obj-pub.aarch64-unknown-linux-gnu.h src/lock-obj-pub.native.h + fi + + make "${_MAKE_FLAGS}" -C src code-to-errno.h + make "${_MAKE_FLAGS}" -C src code-from-errno.h + make "${_MAKE_FLAGS}" -C src gpg-error.h + make "${_MAKE_FLAGS}" -C src libgpg-error.la + + make -C src install-nodist_includeHEADERS install-pkgconfigDATA \ + install-m4dataDATA install-binSCRIPTS install-libLTLIBRARIES + return $? +} + +function build_libgcrypt() { + echo "Building libgcrypt" + cd "${GCRYPT_SRC}" + ./configure "${_CONFIGURE_FLAGS[@]}" "${_CONF_STATIC[@]}" \ + --disable-doc --with-pic "${_GCRYPT_CONF_FLAGS}" \ + --with-libgpg-error-prefix="${_PREFIX}" + + make "${_MAKE_FLAGS}" -C cipher libcipher.la + make "${_MAKE_FLAGS}" -C random librandom.la + make "${_MAKE_FLAGS}" -C mpi libmpi.la + make "${_MAKE_FLAGS}" -C compat libcompat.la + + make "${_MAKE_FLAGS}" -C src libgcrypt.la + + make -C src install-nodist_includeHEADERS \ + install-m4dataDATA install-binSCRIPTS install-libLTLIBRARIES + return $? +} + +function build_libotr() { + local _f + + echo "Building libotr" + cd "${OTR_SRC}" + + aclocal -I "${_PREFIX}/share/aclocal" + autoconf + automake + + ./configure "${_CONFIGURE_FLAGS[@]}" --enable-shared --with-pic \ + --with-libgcrypt-prefix="${_PREFIX}" + + # libtool archive (*.la) files are the devil's work + rm -f "${_PREFIX}"/lib/*.la + sed -i 's|^hardcode_libdir_flag_spec=.*|hardcode_libdir_flag_spec=""|g' libtool + sed -i 's|^runpath_var=LD_RUN_PATH|runpath_var=DIE_RPATH_DIE|g' libtool + + make "${_MAKE_FLAGS}" -C src + + case "${_TARGET_OS}" in + win*) + cd src + "${CC}" -static-libgcc -s -shared -Wl,-no-undefined "${LDFLAGS[@]}" -o libotr.dll \ + ./*.o \ + -L"${_PREFIX}/lib" "${_PREFIX}/lib/libgcrypt.a" "${_PREFIX}/lib/libgpg-error.a" \ + -L"${_LIBDIR}" -lws2_32 -lssp + cp libotr.dll "${_PREFIX}/bin" + ;; + linux*) + cd src + "${CC}" -shared "${LDFLAGS[@]}" -Wl,-soname -Wl,libotr.so \ + .libs/*.o \ + -L"${_PREFIX}/lib" "${_PREFIX}/lib/libgcrypt.a" "${_PREFIX}/lib/libgpg-error.a" \ + --sysroot="${MOZ_FETCHES_DIR}/${SYSROOT}" \ + -Wl,-soname -Wl,libotr.so -o libotr.so + cp libotr.so "${_PREFIX}/lib" + ;; + macos*) + cd src + "${CC}" -dynamiclib -Wl,-flat_namespace -Wl,-undefined -Wl,suppress -o libotr.dylib \ + .libs/*.o \ + "-L${_PREFIX}/lib" "${_PREFIX}/lib/libgcrypt.a" "${_PREFIX}/lib/libgpg-error.a" \ + -isysroot "${MACOS_SDK_DIR}" \ + -install_name "@executable_path/libotr.dylib" \ + -compatibility_version 7 -current_version 7.1 -Wl,-single_module + cp libotr.dylib "${_PREFIX}/lib" + esac + + return $? +} + +function package_libotr_artifact() { + local _f + + cd "${BUILD}" + rm -rf "${_ARTIFACT_STAGEDIR}" + + mkdir "${_ARTIFACT_STAGEDIR}" + + for _f in ${_TARGET_LIBS}; do + install "${_INSTDIR}/${_f}" "${_ARTIFACT_STAGEDIR}" + done + case "${_TARGET_OS}" in + win*) + install "${_LIBDIR}/libssp-0.dll" "${_ARTIFACT_STAGEDIR}" + ;; + esac + + rm -rf "${UPLOAD_DIR}" && mkdir -p "${UPLOAD_DIR}" + TARFILE="${UPLOAD_DIR}/libotr.tar.${COMPRESS_EXT}" + tar -acf "${TARFILE}" -C "${_ARTIFACT_STAGEDIR}" . + + return 0 +} + +# variables specific to an arch, but apply to all dependencies +case "${_TARGET_OS}" in + win32) + export PATH="${MOZ_FETCHES_DIR}/mingw32/bin:$PATH" + export _TARGET_TRIPLE="i686-w64-mingw32" + export CC="${_TARGET_TRIPLE}-gcc" + _LIBDIR="/usr/lib/gcc/${_TARGET_TRIPLE}/10-win32" + export LDFLAGS="-L${_LIBDIR}" + _OS_CONFIGURE_FLAGS=(--host="${_TARGET_TRIPLE}" --target="${_TARGET_TRIPLE}") + _CONF_STATIC=(--enable-static --enable-shared) + + _TARGET_LIBS="bin/libotr.dll" + ;; + win64) + export PATH="${MOZ_FETCHES_DIR}/mingw32/bin:$PATH" + export _TARGET_TRIPLE="x86_64-w64-mingw32" + export CC="${_TARGET_TRIPLE}-gcc" + _LIBDIR="/usr/lib/gcc/${_TARGET_TRIPLE}/10-win32" + export LDFLAGS="-L${_LIBDIR}" + _OS_CONFIGURE_FLAGS=(--host="${_TARGET_TRIPLE}" --target="${_TARGET_TRIPLE}") + _CONF_STATIC=(--enable-static --enable-shared) + + _TARGET_LIBS="bin/libotr.dll" + ;; + macosx64) + for _t in cctools/bin clang/bin binutils/bin; do + PATH="${MOZ_FETCHES_DIR}/${_t}:$PATH" + done + export PATH + + export _TARGET_TRIPLE="x86_64-apple-darwin" + export MACOS_SDK_DIR="${MOZ_FETCHES_DIR}/${MACOS_SDK_DIR}" + export CROSS_PRIVATE_FRAMEWORKS="${MACOS_SDK_DIR}/System/Library/PrivateFrameworks" + export CROSS_SYSROOT="${MACOS_SDK_DIR}" + + export CC="${_TARGET_TRIPLE}-clang" + export LD="${_TARGET_TRIPLE}-ld" + export CFLAGS="-isysroot ${CROSS_SYSROOT} -mmacosx-version-min=${MACOS_TARGET_SDK}" + export LDFLAGS="-isysroot ${CROSS_SYSROOT}" + export DSYMUTIL="${MOZ_FETCHES_DIR}/llvm-dsymutil/llvm-dsymutil" + + _OS_CONFIGURE_FLAGS=(--host="${_TARGET_TRIPLE}" --target="${_TARGET_TRIPLE}") + _GCRYPT_CONF_FLAGS="--disable-asm" + _CONF_STATIC=(--enable-static --disable-shared) + + _TARGET_LIBS="lib/libotr.dylib" + ;; + macosx64-aarch64) + for _t in cctools/bin clang/bin binutils/bin; do + PATH="${MOZ_FETCHES_DIR}/${_t}:$PATH" + done + export PATH + + export _TARGET_TRIPLE="aarch64-apple-darwin" + export MACOS_SDK_DIR="${MOZ_FETCHES_DIR}/${MACOS_SDK_DIR}" + export CROSS_PRIVATE_FRAMEWORKS="${MACOS_SDK_DIR}/System/Library/PrivateFrameworks" + export CROSS_SYSROOT="${MACOS_SDK_DIR}" + + export CC="${_TARGET_TRIPLE}-clang" + export LD="${_TARGET_TRIPLE}-ld" + export CFLAGS="-isysroot ${CROSS_SYSROOT} -mmacosx-version-min=${MACOS_TARGET_SDK}" + export LDFLAGS="-isysroot ${CROSS_SYSROOT}" + export DSYMUTIL="${MOZ_FETCHES_DIR}/llvm-dsymutil/llvm-dsymutil" + + _OS_CONFIGURE_FLAGS=(--host="${_TARGET_TRIPLE}" --target="${_TARGET_TRIPLE}") + _GCRYPT_CONF_FLAGS="--disable-asm" + _CONF_STATIC=(--enable-static --disable-shared) + + _TARGET_LIBS="lib/libotr.dylib" + ;; + linux32) + for _t in clang/bin binutils/bin; do + PATH="${MOZ_FETCHES_DIR}/${_t}:$PATH" + done + export PATH + + SYSROOT="sysroot-i686-linux-gnu" + export _TARGET_TRIPLE="i686-pc-linux" + export CC="i686-linux-gnu-clang" + export CFLAGS="--sysroot=${MOZ_FETCHES_DIR}/${SYSROOT}" + export CCASFLAGS="--sysroot=${MOZ_FETCHES_DIR}/${SYSROOT}" + export LDFLAGS="--target=${_TARGET_TRIPLE} -m32 -march=pentium-m -msse -msse2 -mfpmath=sse -fuse-ld=lld" + + export AR=llvm-ar + export RANLIB=llvm-ranlib + export NM=llvm-nm + export STRIP=llvm-strip + + _OS_CONFIGURE_FLAGS=(--host="${_TARGET_TRIPLE}" --target="${_TARGET_TRIPLE}") + _OS_CONFIGURE_FLAGS+=(--with-sysroot="${MOZ_FETCHES_DIR}/${SYSROOT}") + _CONF_STATIC=(--enable-static --disable-shared) + _TARGET_LIBS="lib/libotr.so" + ;; + linux64) + for _t in clang/bin binutils/bin; do + PATH="${MOZ_FETCHES_DIR}/${_t}:$PATH" + done + export PATH + + SYSROOT="sysroot-x86_64-linux-gnu" + export _TARGET_TRIPLE="x86_64-pc-linux" + export CC="clang" + export CFLAGS="--sysroot=${MOZ_FETCHES_DIR}/${SYSROOT}" + export CASFLAGS="--sysroot=${MOZ_FETCHES_DIR}/${SYSROOT}" + export LDFLAGS="-fuse-ld=lld" + export AR=llvm-ar + export RANLIB=llvm-ranlib + export NM=llvm-nm + export STRIP=llvm-strip + + _OS_CONFIGURE_FLAGS=(--host="${_TARGET_TRIPLE}" --target="${_TARGET_TRIPLE}") + _OS_CONFIGURE_FLAGS+=(--with-sysroot="${MOZ_FETCHES_DIR}/${SYSROOT}") + _CONF_STATIC=(--enable-static --disable-shared) + _TARGET_LIBS="lib/libotr.so" + ;; + linux-aarch64) + for _t in clang/bin binutils/bin; do + PATH="${MOZ_FETCHES_DIR}/${_t}:$PATH" + done + export PATH + + SYSROOT="sysroot-aarch64-linux-gnu" + export _TARGET_TRIPLE="aarch64-pc-linux" + export CC="aarch64-linux-gnu-clang" + export CFLAGS="--sysroot=${MOZ_FETCHES_DIR}/${SYSROOT}" + export CASFLAGS="--sysroot=${MOZ_FETCHES_DIR}/${SYSROOT}" + export LDFLAGS="-fuse-ld=lld" + export AR=llvm-ar + export RANLIB=llvm-ranlib + export NM=llvm-nm + export OBJDUMP=llvm-objdump + export STRIP=llvm-strip + + _OS_CONFIGURE_FLAGS=(--host="${_TARGET_TRIPLE}" --target="${_TARGET_TRIPLE}") + _OS_CONFIGURE_FLAGS+=(--with-sysroot="${MOZ_FETCHES_DIR}/${SYSROOT}") + _CONF_STATIC=(--enable-static --disable-shared) + _TARGET_LIBS="lib/libotr.so" + ;; + *) + echo "Invalid target platform: ${_TARGET_OS}" + exit 1 + ;; +esac + +_CONFIGURE_FLAGS=("${_BASE_CONFIGURE[@]}" "${_OS_CONFIGURE_FLAGS[@]}") +_MAKE_FLAGS="-j$(nproc)" + +# Basic dependency structure. +# Build block, followed by packaging block. +# Each step in a block depends on the previous completing successfully. +# The packaging block depends on the build block's success. +{ + copy_sources && + clang_cfg && + build_libgpg-error && + build_libgcrypt && + build_libotr +} && { + package_libotr_artifact +} && exit 0 + +# Ideally, the "exit 0" above ran after the packaging block ran successfully. +# In case it didn't, error out here so CI catches it. +exit 1 diff --git a/comm/taskcluster/scripts/build-source-docs.sh b/comm/taskcluster/scripts/build-source-docs.sh new file mode 100755 index 0000000000..e8b104b448 --- /dev/null +++ b/comm/taskcluster/scripts/build-source-docs.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +set -xe + +# duplicate the functionality of taskcluster-lib-urls, but in bash.. +queue_base="$TASKCLUSTER_PROXY_URL/api/queue/v1" + +# Get RTD secret location from task definition +if [ -n "${TASK_ID}" ]; then + curl --location --retry 10 --retry-delay 10 -o /builds/worker/task.json "$queue_base/task/$TASK_ID" + RTD_SECRET=$(jq -r '.scopes[] | select(contains ("rtd-webhook"))' /builds/worker/task.json | awk -F: '{print $3}') +fi + +# Get the secret value from the secrets service +if [ -n "${RTD_SECRET}" ] && getent hosts taskcluster; then + set +x # Don't echo these + secrets_url="${TASKCLUSTER_PROXY_URL}/api/secrets/v1/secret/${RTD_SECRET}" + SECRET=$(curl "${secrets_url}") + TOKEN=$(echo "${SECRET}" | jq -r '.secret.token') +elif [ -n "${RTD_TOKEN}" ]; then # Allow for local testing. + TOKEN="${RTD_TOKEN}" +fi + +if [ -n "${TOKEN}" ]; then + curl \ + -X POST \ + -d "branches=latest" \ + -d "token=${TOKEN}" \ + https://readthedocs.com/api/v2/webhook/thunderbird-thunderbird-source-docs/9778/ +fi + diff --git a/comm/taskcluster/scripts/desktop_comm_l10n.py b/comm/taskcluster/scripts/desktop_comm_l10n.py new file mode 100755 index 0000000000..ae395e50d1 --- /dev/null +++ b/comm/taskcluster/scripts/desktop_comm_l10n.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import sys + +GECKO_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")) +MOZHARNESS = os.path.join(GECKO_PATH, "testing/mozharness") +COMM_PYTHON_L10N = os.path.join(GECKO_PATH, "comm/python/l10n") +sys.path.insert(1, MOZHARNESS) +sys.path.insert(1, COMM_PYTHON_L10N) + +from zstandard import ZstdCompressor + +from mozharness.base.script import BaseScript +from mozharness.base.vcs.vcsbase import VCSMixin +from mozharness.mozilla.automation import AutomationMixin +from mozharness.mozilla.l10n.locales import LocalesMixin +from mozpack.archive import create_tar_from_files +from mozpack.copier import FileRegistry +from mozpack.files import FileFinder +from tbxchannel.l10n_merge import COMM_STRINGS_PATTERNS, GECKO_STRINGS_PATTERNS + + +class CommMultiLocale(LocalesMixin, AutomationMixin, VCSMixin, BaseScript): + config_options = [ + [ + [ + "--locale-list", + ], + { + "action": "store", + "dest": "locale_list", + "type": "string", + "help": "File with locales to include. Either all-locales or shipped-locales", + }, + ], + [ + [ + "--comm-locales-file", + ], + { + "action": "store", + "dest": "hg_comm_locales_file", + "type": "string", + "help": "File with HG revision of comm-l10n monorepo to use", + }, + ], + [ + [ + "--browser-locales-file", + ], + { + "action": "store", + "dest": "locales_file", + "type": "string", + "help": "File with HG revisions of l10n-central repositories", + }, + ], + ] + + def __init__(self, require_config_file=False): + buildscript_kwargs = { + "all_actions": [ + "clone-gecko-locales", + "clone-monorepo", + "merge-repos", + "pack-merged", + "gen-changesets", + ], + "config": { + "ignore_locales": ["en-US"], + "log_name": "multi_locale", + "hg_merged_dir": "l10n_merged", + "objdir": "obj-build", + "upload_file": "strings_all.tar.zst", + "changesets_file": "l10n-changesets.json", + }, + } + + LocalesMixin.__init__(self) + BaseScript.__init__( + self, + config_options=self.config_options, + require_config_file=require_config_file, + **buildscript_kwargs, + ) + self.upload_env = None + self.file_registry = None + self.comm_l10n_revision = None + + def query_abs_dirs(self): + if self.abs_dirs: + return self.abs_dirs + abs_dirs = super(CommMultiLocale, self).query_abs_dirs() + c = self.config + dirs = {} + dirs["abs_checkout_dir"] = os.path.abspath(os.path.join(abs_dirs["abs_src_dir"], "..")) + dirs["abs_work_dir"] = os.path.join(c["base_work_dir"], c["work_dir"]) + # Needs to match abs_dirs["abs_l10n_dir"] set in mozharness.mozilla.l10n.locales + dirs["abs_l10n_central_dir"] = os.path.abspath( + os.path.join(dirs["abs_checkout_dir"], "l10n-central") + ) + dirs["abs_comm_l10n_dir"] = os.path.abspath( + os.path.join(dirs["abs_checkout_dir"], "comm-l10n") + ) + dirs["abs_merged_dir"] = os.path.abspath( + os.path.join(dirs["abs_work_dir"], "l10n-central") + ) + for key in dirs.keys(): + if key not in abs_dirs: + abs_dirs[key] = dirs[key] + self.abs_dirs = abs_dirs + return self.abs_dirs + + def _query_upload_env(self): + """returns the environment used for the upload step""" + if self.upload_env: + return self.upload_env + config = self.config + + upload_env = self.query_env(partial_env=config.get("upload_env")) + + self.upload_env = upload_env + return self.upload_env + + def _ensure_upload_path(self): + env = self._query_upload_env() + if "UPLOAD_PATH" in env and not os.path.exists(env["UPLOAD_PATH"]): + self.mkdir_p(env["UPLOAD_PATH"]) + + def get_gecko_l10n_revisions(self): + # Populate self.locales with Thunderbird's locales, and revisions + # from browser/locales/l10n-changesets.json + c = self.config + ignore_locales = c.get("ignore_locales", []) + + dirs = self.query_abs_dirs() + locale_list = os.path.join(dirs["abs_src_dir"], c["locale_list"]) + locales = self.parse_locales_file(locale_list) + locale_changesets_file = os.path.join(dirs["abs_src_dir"], c["locales_file"]) + # parse_locales_file fills in self.l10n_revisions with changesets + self.parse_locales_file(locale_changesets_file) + + for locale in ignore_locales: + if locale in locales: + self.debug("Ignoring locale %s." % locale) + locales.remove(locale) + + self.locales = locales + + # Actions {{{2 + def clone_gecko_locales(self): + self.get_gecko_l10n_revisions() + self.pull_locale_source() + + def clone_monorepo(self): + c = self.config + dirs = self.query_abs_dirs() + + locales_file = os.path.join(dirs["abs_src_dir"], c["hg_comm_locales_file"]) + locales_data = {} + if locales_file.endswith(".json"): + with open(locales_file) as fh: + locales_data = json.load(fh) + # would use en-US, but it's not in this file! + self.comm_l10n_revision = locales_data.get("en-GB", {}).get("revision") + + if self.comm_l10n_revision: + self.mkdir_p(dirs["abs_checkout_dir"]) + self.vcs_checkout( + dest=dirs["abs_comm_l10n_dir"], + repo=c["hg_comm_l10n_repo"], + branch=self.comm_l10n_revision, + vcs="hg", + ) + else: + raise Exception( + f"Unable to find revision from comm-l10n repo using " + f"{c['hg_comm_locales_file']}." + ) + + def merge_repos(self): + dirs = self.query_abs_dirs() + if os.path.exists(dirs["abs_merged_dir"]): + self.rmtree(dirs["abs_merged_dir"]) + + file_registry = FileRegistry() + + def add_to_registry(base_path, patterns): + finder = FileFinder(base_path) + for pattern in patterns: + for _lang in self.locales: + for _filepath, _fileobj in finder.find(pattern.format(lang=_lang)): + _filepath = os.path.join("l10n-central", _filepath) + file_registry.add(_filepath, _fileobj) + + add_to_registry(dirs["abs_l10n_central_dir"], GECKO_STRINGS_PATTERNS) + add_to_registry(dirs["abs_comm_l10n_dir"], COMM_STRINGS_PATTERNS) + + self.file_registry = file_registry + + def pack_merged(self): + self._ensure_upload_path() + upload_path = self.config["upload_env"]["UPLOAD_PATH"] + archive_path = os.path.join(upload_path, self.config["upload_file"]) + + with open(archive_path, "wb") as f: + with ZstdCompressor().stream_writer(f) as z: + create_tar_from_files(z, dict(self.file_registry)) + + def gen_changesets(self): + changeset_data = { + "gecko_strings": self.gecko_locale_revisions, + "comm_strings": { + "repo": self.config["hg_comm_l10n_repo"], + "revision": self.comm_l10n_revision, + }, + } + upload_path = self.config["upload_env"]["UPLOAD_PATH"] + changesets_file = os.path.join(upload_path, self.config["changesets_file"]) + with open(changesets_file, "w") as f: + json.dump(changeset_data, f, sort_keys=True, indent=2) + + +if __name__ == "__main__": + single_locale = CommMultiLocale() + single_locale.run_and_exit() diff --git a/comm/taskcluster/scripts/source-test-clang-format.sh b/comm/taskcluster/scripts/source-test-clang-format.sh new file mode 100755 index 0000000000..90f449a03b --- /dev/null +++ b/comm/taskcluster/scripts/source-test-clang-format.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e + +# Run the Firefox setup script +source "$HOME/checkouts/gecko/taskcluster/scripts/misc/source-test-clang-setup.sh" + +# Append comm/.clang-format-ignore contents to $topsrcdir/.clang-format-ignore +sed -e 's%^\([a-z]\)%comm/\1%' comm/.clang-format-ignore >> .clang-format-ignore + +# Update mozconfig file with Thunderbird build options +cat <> "$MOZCONFIG" +ac_add_options --enable-project=comm/mail +EOT + +# Run mach clang-format +# shellcheck disable=SC2068 +./mach --log-no-times clang-format --output "$HOME/clang-format.json" --format json -p $@ +# shellcheck disable=SC2068 +./mach --log-no-times clang-format --output "$HOME/clang-format.diff" --format diff -p $@ + +# Exit with an error code if clang-format.diff contains a proper diff. +# Needed because mach clang-format will exit 0 regardless of outcome. +# If no formatting is needed, clang-format.diff will have a single \n, +# so check for a file size > 1 byte. +DIFF_SIZE=$(stat -c %s "$HOME/clang-format.diff") +if [[ "$DIFF_SIZE" -gt 1 ]]; then + echo "Exiting with error status. DIFF_SIZE is $DIFF_SIZE." + exit 1 +else + exit 0 +fi -- cgit v1.2.3