summaryrefslogtreecommitdiffstats
path: root/comm/taskcluster/scripts
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 17:32:43 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 17:32:43 +0000
commit6bf0a5cb5034a7e684dcc3500e841785237ce2dd (patch)
treea68f146d7fa01f0134297619fbe7e33db084e0aa /comm/taskcluster/scripts
parentInitial commit. (diff)
downloadthunderbird-6bf0a5cb5034a7e684dcc3500e841785237ce2dd.tar.xz
thunderbird-6bf0a5cb5034a7e684dcc3500e841785237ce2dd.zip
Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'comm/taskcluster/scripts')
-rw-r--r--comm/taskcluster/scripts/are-we-esmified-yet.py147
-rwxr-xr-xcomm/taskcluster/scripts/build-l10n-pre.sh90
-rwxr-xr-xcomm/taskcluster/scripts/build-libotr.sh360
-rwxr-xr-xcomm/taskcluster/scripts/build-source-docs.sh34
-rwxr-xr-xcomm/taskcluster/scripts/desktop_comm_l10n.py231
-rwxr-xr-xcomm/taskcluster/scripts/source-test-clang-format.sh32
6 files changed, 894 insertions, 0 deletions
diff --git a/comm/taskcluster/scripts/are-we-esmified-yet.py b/comm/taskcluster/scripts/are-we-esmified-yet.py
new file mode 100644
index 0000000000..20eaf108fd
--- /dev/null
+++ b/comm/taskcluster/scripts/are-we-esmified-yet.py
@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+import pathlib
+import subprocess
+import sys
+
+TBPL_FAILURE = 2
+
+excluded_prefix = [
+ "suite/",
+]
+EXCLUSION_FILES = [
+ os.path.join("tools", "lint", "ThirdPartyPaths.txt"),
+]
+
+
+if not (pathlib.Path(".hg").exists() and pathlib.Path("mail/moz.configure").exists()):
+ print(
+ "This script needs to be run inside mozilla-central + comm-central "
+ "checkout of mercurial. "
+ )
+ sys.exit(TBPL_FAILURE)
+
+
+def load_exclusion_files():
+ for path in EXCLUSION_FILES:
+ with open(path, "r") as f:
+ for line in f:
+ excluded_prefix.append(line.strip())
+
+
+def is_excluded(path):
+ """Returns true if the JSM file shouldn't be converted to ESM."""
+ path_str = str(path)
+ for prefix in excluded_prefix:
+ if path_str.startswith(prefix):
+ return True
+
+ return False
+
+
+def new_files_struct():
+ return {
+ "jsm": [],
+ "esm": [],
+ "subdir": {},
+ }
+
+
+def put_file(files, kind, path):
+ """Put a path into files tree structure."""
+
+ if is_excluded(path):
+ return
+
+ name = path.name
+
+ current_files = files
+ for part in path.parent.parts:
+ if part not in current_files["subdir"]:
+ current_files["subdir"][part] = new_files_struct()
+ current_files = current_files["subdir"][part]
+
+ current_files[kind].append(name)
+
+
+def run(cmd):
+ """Run command and return output as lines, excluding empty line."""
+ lines = subprocess.run(cmd, stdout=subprocess.PIPE).stdout.decode()
+ return filter(lambda x: x != "", lines.split("\n"))
+
+
+def collect_jsm(files):
+ """Collect JSM files."""
+ kind = "jsm"
+
+ # jsm files
+ cmd = ["hg", "files", "set:glob:**/*.jsm"]
+ for line in run(cmd):
+ put_file(files, kind, pathlib.Path(line))
+
+ # js files with EXPORTED_SYMBOLS
+ cmd = ["hg", "files", "set:grep('EXPORTED_SYMBOLS = \[') and glob:**/*.js"]
+ for line in run(cmd):
+ put_file(files, kind, pathlib.Path(line))
+
+
+def collect_esm(files):
+ """Collect system ESM files."""
+ kind = "esm"
+
+ # sys.mjs files
+ cmd = ["hg", "files", "set:glob:**/*.sys.mjs"]
+
+ for line in run(cmd):
+ put_file(files, kind, pathlib.Path(line))
+
+
+def to_stat(files):
+ """Convert files tree into status tree."""
+ jsm = len(files["jsm"])
+ esm = len(files["esm"])
+ subdir = {}
+
+ for key, sub_files in files["subdir"].items():
+ sub_stat = to_stat(sub_files)
+
+ subdir[key] = sub_stat
+ jsm += sub_stat["jsm"]
+ esm += sub_stat["esm"]
+
+ stat = {
+ "jsm": jsm,
+ "esm": esm,
+ }
+ if len(subdir):
+ stat["subdir"] = subdir
+
+ return stat
+
+
+def main():
+ cmd = ["hg", "parent", "--template", "{node}"]
+ commit_hash = list(run(cmd))[0]
+
+ cmd = ["hg", "parent", "--template", "{date|shortdate}"]
+ date = list(run(cmd))[0]
+
+ files = new_files_struct()
+ collect_jsm(files)
+ collect_esm(files)
+
+ stat = to_stat(files)
+ stat["hash"] = commit_hash
+ stat["date"] = date
+
+ print(json.dumps(stat, indent=2))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/comm/taskcluster/scripts/build-l10n-pre.sh b/comm/taskcluster/scripts/build-l10n-pre.sh
new file mode 100755
index 0000000000..7482dc7c5a
--- /dev/null
+++ b/comm/taskcluster/scripts/build-l10n-pre.sh
@@ -0,0 +1,90 @@
+#! /bin/bash -vex
+
+set -x -e
+
+echo "running as" $(id)
+
+####
+# Taskcluster friendly wrapper for performing fx desktop l10n repacks via mozharness.
+# Based on ./build-l10n.sh
+####
+
+# Inputs, with defaults
+
+: MOZHARNESS_SCRIPT ${MOZHARNESS_SCRIPT}
+: MOZHARNESS_CONFIG ${MOZHARNESS_CONFIG}
+: MOZHARNESS_CONFIG_PATHS ${MOZHARNESS_CONFIG_PATHS}
+: MOZHARNESS_ACTIONS ${MOZHARNESS_ACTIONS}
+: MOZHARNESS_OPTIONS ${MOZHARNESS_OPTIONS}
+
+: TOOLTOOL_CACHE ${TOOLTOOL_CACHE:=/builds/worker/tooltool-cache}
+
+: MOZ_SCM_LEVEL ${MOZ_SCM_LEVEL:=1}
+
+: MOZ_SCM_LEVEL ${MOZ_SCM_LEVEL:=1}
+
+: WORKSPACE ${WORKSPACE:=/builds/worker/workspace}
+: MOZ_OBJDIR ${MOZ_OBJDIR:=$WORKSPACE/obj-build}
+
+set -v
+
+fail() {
+ echo # make sure error message is on a new line
+ echo "[build-l10n-pre.sh:error]" "${@}"
+ exit 1
+}
+
+export MOZ_CRASHREPORTER_NO_REPORT=1
+export TINDERBOX_OUTPUT=1
+
+# test required parameters are supplied
+if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi
+if [[ -z "${MOZHARNESS_CONFIG}" && -z "${EXTRA_MOZHARNESS_CONFIG}" ]]; then fail "MOZHARNESS_CONFIG or EXTRA_MOZHARNESS_CONFIG is not set"; fi
+
+# set up mozharness configuration, via command line, env, etc.
+
+# $TOOLTOOL_CACHE bypasses mozharness completely and is read by tooltool_wrapper.sh to set the
+# cache. However, only some mozharness scripts use tooltool_wrapper.sh, so this may not be
+# entirely effective.
+export TOOLTOOL_CACHE
+
+export MOZ_OBJDIR
+
+config_path_cmds=""
+for path in ${MOZHARNESS_CONFIG_PATHS}; do
+ config_path_cmds="${config_path_cmds} --extra-config-path ${GECKO_PATH}/${path}"
+done
+
+# support multiple, space delimited, config files
+config_cmds=""
+for cfg in $MOZHARNESS_CONFIG; do
+ config_cmds="${config_cmds} --config ${cfg}"
+done
+
+# if MOZHARNESS_ACTIONS is given, only run those actions (completely overriding default_actions
+# in the mozharness configuration)
+if [ -n "$MOZHARNESS_ACTIONS" ]; then
+ actions=""
+ for action in $MOZHARNESS_ACTIONS; do
+ actions="$actions --$action"
+ done
+fi
+
+# if MOZHARNESS_OPTIONS is given, append them to mozharness command line run
+if [ -n "$MOZHARNESS_OPTIONS" ]; then
+ options=""
+ for option in $MOZHARNESS_OPTIONS; do
+ options="$options --$option"
+ done
+fi
+
+cd /builds/worker
+
+$GECKO_PATH/mach python -- \
+ $GECKO_PATH/${MOZHARNESS_SCRIPT} \
+ ${config_path_cmds} \
+ ${config_cmds} \
+ $actions \
+ $options \
+ --log-level=debug \
+ --work-dir=$WORKSPACE \
diff --git a/comm/taskcluster/scripts/build-libotr.sh b/comm/taskcluster/scripts/build-libotr.sh
new file mode 100755
index 0000000000..60df92c531
--- /dev/null
+++ b/comm/taskcluster/scripts/build-libotr.sh
@@ -0,0 +1,360 @@
+#!/bin/bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+set -x -eE -v
+
+_TARGET_OS="$1"
+
+# Environment variables that are set by Taskcluster.
+GECKO_PATH=${GECKO_PATH:-"/builds/worker/workspace/build/src"}
+MOZ_FETCHES_DIR=${MOZ_FETCHES_DIR:-"/builds/worker/fetches"}
+UPLOAD_DIR=${UPLOAD_DIR:-"/builds/worker/artifacts"}
+WORKSPACE=${WORKSPACE:-"${HOME}/workspace"}
+MACOS_SDK_DIR=${MACOS_SDK_DIR:-"MacOSX11.3.sdk"}
+MACOS_TARGET_SDK=${MACOS_TARGET_SDK:-"10.12"}
+
+
+# Set $DEVEL_TESTING during script development on a local machine
+if [[ -n ${DEVEL_TESTING} ]]; then
+ rm -rf "${UPLOAD_DIR}" "${WORKSPACE}"
+ mkdir "${UPLOAD_DIR}" "${WORKSPACE}"
+fi
+
+cd "$WORKSPACE"
+if [[ ! -d build ]]; then
+ mkdir build
+fi
+for _d in build/libgpg-error build/libgcrypt build/libotr build/build_prefix; do
+ if [[ -e "${_d}" ]]; then
+ rm -rf "${_d}"
+ fi
+done
+BUILD="${WORKSPACE}/build"
+
+COMPRESS_EXT=xz
+
+_INSTDIR="build_prefix"
+_PREFIX="${BUILD}/${_INSTDIR}"
+
+_ARTIFACT_STAGEDIR="${BUILD}/libotr_stage"
+
+THIRD_PARTY_SRC="${GECKO_PATH}/comm/third_party"
+
+GPG_ERROR_SRC="${BUILD}/libgpg-error"
+GCRYPT_SRC="${BUILD}/libgcrypt"
+OTR_SRC="${BUILD}/libotr"
+
+# Set environment variables needed for all dependencies
+_BASE_CONFIGURE=(--build=x86_64-pc-linux --prefix="${_PREFIX}" --disable-silent-rules)
+
+_OS_CONFIGURE_FLAGS=() # May be overridden per target OS
+
+
+function clang_cfg() {
+ # autotools and friends seem to work better with Clang if the compiler
+ # is named <target>-clang. This applies to macOS only. It does not seem
+ # necessary when building for Linux.
+ local _i _clang_cfg_dir _clang_dir
+
+ _clang_cfg_dir="${THIRD_PARTY_SRC}/clang"
+ _clang_dir="${MOZ_FETCHES_DIR}/clang/bin"
+
+ cp -a "${_clang_cfg_dir}"/*.cfg "${_clang_dir}"
+ for _i in x86_64-apple-darwin aarch64-apple-darwin aarch64-linux-gnu i686-linux-gnu; do
+ ln -s clang "${_clang_dir}/${_i}-clang"
+ done
+ return 0
+}
+
+function copy_sources() {
+ # The checkout directory should be treated readonly
+ local _pkg
+ cd "${BUILD}"
+ for _pkg in libgpg-error libgcrypt libotr; do
+ cp -a "${THIRD_PARTY_SRC}/${_pkg}" .
+ done
+}
+
+function build_libgpg-error() {
+ echo "Building libgpg-error"
+ cd "${GPG_ERROR_SRC}"
+
+ ./configure "${_CONFIGURE_FLAGS[@]}" "${_CONF_STATIC[@]}" \
+ --disable-tests --disable-doc --with-pic
+
+ # Hack... *sigh*
+ if [[ "${_TARGET_OS}" == "linux-aarch64" ]]; then
+ cp src/syscfg/lock-obj-pub.aarch64-unknown-linux-gnu.h src/lock-obj-pub.native.h
+ fi
+
+ make "${_MAKE_FLAGS}" -C src code-to-errno.h
+ make "${_MAKE_FLAGS}" -C src code-from-errno.h
+ make "${_MAKE_FLAGS}" -C src gpg-error.h
+ make "${_MAKE_FLAGS}" -C src libgpg-error.la
+
+ make -C src install-nodist_includeHEADERS install-pkgconfigDATA \
+ install-m4dataDATA install-binSCRIPTS install-libLTLIBRARIES
+ return $?
+}
+
+function build_libgcrypt() {
+ echo "Building libgcrypt"
+ cd "${GCRYPT_SRC}"
+ ./configure "${_CONFIGURE_FLAGS[@]}" "${_CONF_STATIC[@]}" \
+ --disable-doc --with-pic "${_GCRYPT_CONF_FLAGS}" \
+ --with-libgpg-error-prefix="${_PREFIX}"
+
+ make "${_MAKE_FLAGS}" -C cipher libcipher.la
+ make "${_MAKE_FLAGS}" -C random librandom.la
+ make "${_MAKE_FLAGS}" -C mpi libmpi.la
+ make "${_MAKE_FLAGS}" -C compat libcompat.la
+
+ make "${_MAKE_FLAGS}" -C src libgcrypt.la
+
+ make -C src install-nodist_includeHEADERS \
+ install-m4dataDATA install-binSCRIPTS install-libLTLIBRARIES
+ return $?
+}
+
+function build_libotr() {
+ local _f
+
+ echo "Building libotr"
+ cd "${OTR_SRC}"
+
+ aclocal -I "${_PREFIX}/share/aclocal"
+ autoconf
+ automake
+
+ ./configure "${_CONFIGURE_FLAGS[@]}" --enable-shared --with-pic \
+ --with-libgcrypt-prefix="${_PREFIX}"
+
+ # libtool archive (*.la) files are the devil's work
+ rm -f "${_PREFIX}"/lib/*.la
+ sed -i 's|^hardcode_libdir_flag_spec=.*|hardcode_libdir_flag_spec=""|g' libtool
+ sed -i 's|^runpath_var=LD_RUN_PATH|runpath_var=DIE_RPATH_DIE|g' libtool
+
+ make "${_MAKE_FLAGS}" -C src
+
+ case "${_TARGET_OS}" in
+ win*)
+ cd src
+ "${CC}" -static-libgcc -s -shared -Wl,-no-undefined "${LDFLAGS[@]}" -o libotr.dll \
+ ./*.o \
+ -L"${_PREFIX}/lib" "${_PREFIX}/lib/libgcrypt.a" "${_PREFIX}/lib/libgpg-error.a" \
+ -L"${_LIBDIR}" -lws2_32 -lssp
+ cp libotr.dll "${_PREFIX}/bin"
+ ;;
+ linux*)
+ cd src
+ "${CC}" -shared "${LDFLAGS[@]}" -Wl,-soname -Wl,libotr.so \
+ .libs/*.o \
+ -L"${_PREFIX}/lib" "${_PREFIX}/lib/libgcrypt.a" "${_PREFIX}/lib/libgpg-error.a" \
+ --sysroot="${MOZ_FETCHES_DIR}/${SYSROOT}" \
+ -Wl,-soname -Wl,libotr.so -o libotr.so
+ cp libotr.so "${_PREFIX}/lib"
+ ;;
+ macos*)
+ cd src
+ "${CC}" -dynamiclib -Wl,-flat_namespace -Wl,-undefined -Wl,suppress -o libotr.dylib \
+ .libs/*.o \
+ "-L${_PREFIX}/lib" "${_PREFIX}/lib/libgcrypt.a" "${_PREFIX}/lib/libgpg-error.a" \
+ -isysroot "${MACOS_SDK_DIR}" \
+ -install_name "@executable_path/libotr.dylib" \
+ -compatibility_version 7 -current_version 7.1 -Wl,-single_module
+ cp libotr.dylib "${_PREFIX}/lib"
+ esac
+
+ return $?
+}
+
+function package_libotr_artifact() {
+ local _f
+
+ cd "${BUILD}"
+ rm -rf "${_ARTIFACT_STAGEDIR}"
+
+ mkdir "${_ARTIFACT_STAGEDIR}"
+
+ for _f in ${_TARGET_LIBS}; do
+ install "${_INSTDIR}/${_f}" "${_ARTIFACT_STAGEDIR}"
+ done
+ case "${_TARGET_OS}" in
+ win*)
+ install "${_LIBDIR}/libssp-0.dll" "${_ARTIFACT_STAGEDIR}"
+ ;;
+ esac
+
+ rm -rf "${UPLOAD_DIR}" && mkdir -p "${UPLOAD_DIR}"
+ TARFILE="${UPLOAD_DIR}/libotr.tar.${COMPRESS_EXT}"
+ tar -acf "${TARFILE}" -C "${_ARTIFACT_STAGEDIR}" .
+
+ return 0
+}
+
+# variables specific to an arch, but apply to all dependencies
+case "${_TARGET_OS}" in
+ win32)
+ export PATH="${MOZ_FETCHES_DIR}/mingw32/bin:$PATH"
+ export _TARGET_TRIPLE="i686-w64-mingw32"
+ export CC="${_TARGET_TRIPLE}-gcc"
+ _LIBDIR="/usr/lib/gcc/${_TARGET_TRIPLE}/10-win32"
+ export LDFLAGS="-L${_LIBDIR}"
+ _OS_CONFIGURE_FLAGS=(--host="${_TARGET_TRIPLE}" --target="${_TARGET_TRIPLE}")
+ _CONF_STATIC=(--enable-static --enable-shared)
+
+ _TARGET_LIBS="bin/libotr.dll"
+ ;;
+ win64)
+ export PATH="${MOZ_FETCHES_DIR}/mingw32/bin:$PATH"
+ export _TARGET_TRIPLE="x86_64-w64-mingw32"
+ export CC="${_TARGET_TRIPLE}-gcc"
+ _LIBDIR="/usr/lib/gcc/${_TARGET_TRIPLE}/10-win32"
+ export LDFLAGS="-L${_LIBDIR}"
+ _OS_CONFIGURE_FLAGS=(--host="${_TARGET_TRIPLE}" --target="${_TARGET_TRIPLE}")
+ _CONF_STATIC=(--enable-static --enable-shared)
+
+ _TARGET_LIBS="bin/libotr.dll"
+ ;;
+ macosx64)
+ for _t in cctools/bin clang/bin binutils/bin; do
+ PATH="${MOZ_FETCHES_DIR}/${_t}:$PATH"
+ done
+ export PATH
+
+ export _TARGET_TRIPLE="x86_64-apple-darwin"
+ export MACOS_SDK_DIR="${MOZ_FETCHES_DIR}/${MACOS_SDK_DIR}"
+ export CROSS_PRIVATE_FRAMEWORKS="${MACOS_SDK_DIR}/System/Library/PrivateFrameworks"
+ export CROSS_SYSROOT="${MACOS_SDK_DIR}"
+
+ export CC="${_TARGET_TRIPLE}-clang"
+ export LD="${_TARGET_TRIPLE}-ld"
+ export CFLAGS="-isysroot ${CROSS_SYSROOT} -mmacosx-version-min=${MACOS_TARGET_SDK}"
+ export LDFLAGS="-isysroot ${CROSS_SYSROOT}"
+ export DSYMUTIL="${MOZ_FETCHES_DIR}/llvm-dsymutil/llvm-dsymutil"
+
+ _OS_CONFIGURE_FLAGS=(--host="${_TARGET_TRIPLE}" --target="${_TARGET_TRIPLE}")
+ _GCRYPT_CONF_FLAGS="--disable-asm"
+ _CONF_STATIC=(--enable-static --disable-shared)
+
+ _TARGET_LIBS="lib/libotr.dylib"
+ ;;
+ macosx64-aarch64)
+ for _t in cctools/bin clang/bin binutils/bin; do
+ PATH="${MOZ_FETCHES_DIR}/${_t}:$PATH"
+ done
+ export PATH
+
+ export _TARGET_TRIPLE="aarch64-apple-darwin"
+ export MACOS_SDK_DIR="${MOZ_FETCHES_DIR}/${MACOS_SDK_DIR}"
+ export CROSS_PRIVATE_FRAMEWORKS="${MACOS_SDK_DIR}/System/Library/PrivateFrameworks"
+ export CROSS_SYSROOT="${MACOS_SDK_DIR}"
+
+ export CC="${_TARGET_TRIPLE}-clang"
+ export LD="${_TARGET_TRIPLE}-ld"
+ export CFLAGS="-isysroot ${CROSS_SYSROOT} -mmacosx-version-min=${MACOS_TARGET_SDK}"
+ export LDFLAGS="-isysroot ${CROSS_SYSROOT}"
+ export DSYMUTIL="${MOZ_FETCHES_DIR}/llvm-dsymutil/llvm-dsymutil"
+
+ _OS_CONFIGURE_FLAGS=(--host="${_TARGET_TRIPLE}" --target="${_TARGET_TRIPLE}")
+ _GCRYPT_CONF_FLAGS="--disable-asm"
+ _CONF_STATIC=(--enable-static --disable-shared)
+
+ _TARGET_LIBS="lib/libotr.dylib"
+ ;;
+ linux32)
+ for _t in clang/bin binutils/bin; do
+ PATH="${MOZ_FETCHES_DIR}/${_t}:$PATH"
+ done
+ export PATH
+
+ SYSROOT="sysroot-i686-linux-gnu"
+ export _TARGET_TRIPLE="i686-pc-linux"
+ export CC="i686-linux-gnu-clang"
+ export CFLAGS="--sysroot=${MOZ_FETCHES_DIR}/${SYSROOT}"
+ export CCASFLAGS="--sysroot=${MOZ_FETCHES_DIR}/${SYSROOT}"
+ export LDFLAGS="--target=${_TARGET_TRIPLE} -m32 -march=pentium-m -msse -msse2 -mfpmath=sse -fuse-ld=lld"
+
+ export AR=llvm-ar
+ export RANLIB=llvm-ranlib
+ export NM=llvm-nm
+ export STRIP=llvm-strip
+
+ _OS_CONFIGURE_FLAGS=(--host="${_TARGET_TRIPLE}" --target="${_TARGET_TRIPLE}")
+ _OS_CONFIGURE_FLAGS+=(--with-sysroot="${MOZ_FETCHES_DIR}/${SYSROOT}")
+ _CONF_STATIC=(--enable-static --disable-shared)
+ _TARGET_LIBS="lib/libotr.so"
+ ;;
+ linux64)
+ for _t in clang/bin binutils/bin; do
+ PATH="${MOZ_FETCHES_DIR}/${_t}:$PATH"
+ done
+ export PATH
+
+ SYSROOT="sysroot-x86_64-linux-gnu"
+ export _TARGET_TRIPLE="x86_64-pc-linux"
+ export CC="clang"
+ export CFLAGS="--sysroot=${MOZ_FETCHES_DIR}/${SYSROOT}"
+ export CASFLAGS="--sysroot=${MOZ_FETCHES_DIR}/${SYSROOT}"
+ export LDFLAGS="-fuse-ld=lld"
+ export AR=llvm-ar
+ export RANLIB=llvm-ranlib
+ export NM=llvm-nm
+ export STRIP=llvm-strip
+
+ _OS_CONFIGURE_FLAGS=(--host="${_TARGET_TRIPLE}" --target="${_TARGET_TRIPLE}")
+ _OS_CONFIGURE_FLAGS+=(--with-sysroot="${MOZ_FETCHES_DIR}/${SYSROOT}")
+ _CONF_STATIC=(--enable-static --disable-shared)
+ _TARGET_LIBS="lib/libotr.so"
+ ;;
+ linux-aarch64)
+ for _t in clang/bin binutils/bin; do
+ PATH="${MOZ_FETCHES_DIR}/${_t}:$PATH"
+ done
+ export PATH
+
+ SYSROOT="sysroot-aarch64-linux-gnu"
+ export _TARGET_TRIPLE="aarch64-pc-linux"
+ export CC="aarch64-linux-gnu-clang"
+ export CFLAGS="--sysroot=${MOZ_FETCHES_DIR}/${SYSROOT}"
+ export CASFLAGS="--sysroot=${MOZ_FETCHES_DIR}/${SYSROOT}"
+ export LDFLAGS="-fuse-ld=lld"
+ export AR=llvm-ar
+ export RANLIB=llvm-ranlib
+ export NM=llvm-nm
+ export OBJDUMP=llvm-objdump
+ export STRIP=llvm-strip
+
+ _OS_CONFIGURE_FLAGS=(--host="${_TARGET_TRIPLE}" --target="${_TARGET_TRIPLE}")
+ _OS_CONFIGURE_FLAGS+=(--with-sysroot="${MOZ_FETCHES_DIR}/${SYSROOT}")
+ _CONF_STATIC=(--enable-static --disable-shared)
+ _TARGET_LIBS="lib/libotr.so"
+ ;;
+ *)
+ echo "Invalid target platform: ${_TARGET_OS}"
+ exit 1
+ ;;
+esac
+
+_CONFIGURE_FLAGS=("${_BASE_CONFIGURE[@]}" "${_OS_CONFIGURE_FLAGS[@]}")
+_MAKE_FLAGS="-j$(nproc)"
+
+# Basic dependency structure.
+# Build block, followed by packaging block.
+# Each step in a block depends on the previous completing successfully.
+# The packaging block depends on the build block's success.
+{
+ copy_sources &&
+ clang_cfg &&
+ build_libgpg-error &&
+ build_libgcrypt &&
+ build_libotr
+} && {
+ package_libotr_artifact
+} && exit 0
+
+# Ideally, the "exit 0" above ran after the packaging block ran successfully.
+# In case it didn't, error out here so CI catches it.
+exit 1
diff --git a/comm/taskcluster/scripts/build-source-docs.sh b/comm/taskcluster/scripts/build-source-docs.sh
new file mode 100755
index 0000000000..e8b104b448
--- /dev/null
+++ b/comm/taskcluster/scripts/build-source-docs.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+set -xe
+
+# duplicate the functionality of taskcluster-lib-urls, but in bash..
+queue_base="$TASKCLUSTER_PROXY_URL/api/queue/v1"
+
+# Get RTD secret location from task definition
+if [ -n "${TASK_ID}" ]; then
+ curl --location --retry 10 --retry-delay 10 -o /builds/worker/task.json "$queue_base/task/$TASK_ID"
+ RTD_SECRET=$(jq -r '.scopes[] | select(contains ("rtd-webhook"))' /builds/worker/task.json | awk -F: '{print $3}')
+fi
+
+# Get the secret value from the secrets service
+if [ -n "${RTD_SECRET}" ] && getent hosts taskcluster; then
+ set +x # Don't echo these
+ secrets_url="${TASKCLUSTER_PROXY_URL}/api/secrets/v1/secret/${RTD_SECRET}"
+ SECRET=$(curl "${secrets_url}")
+ TOKEN=$(echo "${SECRET}" | jq -r '.secret.token')
+elif [ -n "${RTD_TOKEN}" ]; then # Allow for local testing.
+ TOKEN="${RTD_TOKEN}"
+fi
+
+if [ -n "${TOKEN}" ]; then
+ curl \
+ -X POST \
+ -d "branches=latest" \
+ -d "token=${TOKEN}" \
+ https://readthedocs.com/api/v2/webhook/thunderbird-thunderbird-source-docs/9778/
+fi
+
diff --git a/comm/taskcluster/scripts/desktop_comm_l10n.py b/comm/taskcluster/scripts/desktop_comm_l10n.py
new file mode 100755
index 0000000000..ae395e50d1
--- /dev/null
+++ b/comm/taskcluster/scripts/desktop_comm_l10n.py
@@ -0,0 +1,231 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+import sys
+
+GECKO_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../.."))
+MOZHARNESS = os.path.join(GECKO_PATH, "testing/mozharness")
+COMM_PYTHON_L10N = os.path.join(GECKO_PATH, "comm/python/l10n")
+sys.path.insert(1, MOZHARNESS)
+sys.path.insert(1, COMM_PYTHON_L10N)
+
+from zstandard import ZstdCompressor
+
+from mozharness.base.script import BaseScript
+from mozharness.base.vcs.vcsbase import VCSMixin
+from mozharness.mozilla.automation import AutomationMixin
+from mozharness.mozilla.l10n.locales import LocalesMixin
+from mozpack.archive import create_tar_from_files
+from mozpack.copier import FileRegistry
+from mozpack.files import FileFinder
+from tbxchannel.l10n_merge import COMM_STRINGS_PATTERNS, GECKO_STRINGS_PATTERNS
+
+
+class CommMultiLocale(LocalesMixin, AutomationMixin, VCSMixin, BaseScript):
+ config_options = [
+ [
+ [
+ "--locale-list",
+ ],
+ {
+ "action": "store",
+ "dest": "locale_list",
+ "type": "string",
+ "help": "File with locales to include. Either all-locales or shipped-locales",
+ },
+ ],
+ [
+ [
+ "--comm-locales-file",
+ ],
+ {
+ "action": "store",
+ "dest": "hg_comm_locales_file",
+ "type": "string",
+ "help": "File with HG revision of comm-l10n monorepo to use",
+ },
+ ],
+ [
+ [
+ "--browser-locales-file",
+ ],
+ {
+ "action": "store",
+ "dest": "locales_file",
+ "type": "string",
+ "help": "File with HG revisions of l10n-central repositories",
+ },
+ ],
+ ]
+
+ def __init__(self, require_config_file=False):
+ buildscript_kwargs = {
+ "all_actions": [
+ "clone-gecko-locales",
+ "clone-monorepo",
+ "merge-repos",
+ "pack-merged",
+ "gen-changesets",
+ ],
+ "config": {
+ "ignore_locales": ["en-US"],
+ "log_name": "multi_locale",
+ "hg_merged_dir": "l10n_merged",
+ "objdir": "obj-build",
+ "upload_file": "strings_all.tar.zst",
+ "changesets_file": "l10n-changesets.json",
+ },
+ }
+
+ LocalesMixin.__init__(self)
+ BaseScript.__init__(
+ self,
+ config_options=self.config_options,
+ require_config_file=require_config_file,
+ **buildscript_kwargs,
+ )
+ self.upload_env = None
+ self.file_registry = None
+ self.comm_l10n_revision = None
+
+ def query_abs_dirs(self):
+ if self.abs_dirs:
+ return self.abs_dirs
+ abs_dirs = super(CommMultiLocale, self).query_abs_dirs()
+ c = self.config
+ dirs = {}
+ dirs["abs_checkout_dir"] = os.path.abspath(os.path.join(abs_dirs["abs_src_dir"], ".."))
+ dirs["abs_work_dir"] = os.path.join(c["base_work_dir"], c["work_dir"])
+ # Needs to match abs_dirs["abs_l10n_dir"] set in mozharness.mozilla.l10n.locales
+ dirs["abs_l10n_central_dir"] = os.path.abspath(
+ os.path.join(dirs["abs_checkout_dir"], "l10n-central")
+ )
+ dirs["abs_comm_l10n_dir"] = os.path.abspath(
+ os.path.join(dirs["abs_checkout_dir"], "comm-l10n")
+ )
+ dirs["abs_merged_dir"] = os.path.abspath(
+ os.path.join(dirs["abs_work_dir"], "l10n-central")
+ )
+ for key in dirs.keys():
+ if key not in abs_dirs:
+ abs_dirs[key] = dirs[key]
+ self.abs_dirs = abs_dirs
+ return self.abs_dirs
+
+ def _query_upload_env(self):
+ """returns the environment used for the upload step"""
+ if self.upload_env:
+ return self.upload_env
+ config = self.config
+
+ upload_env = self.query_env(partial_env=config.get("upload_env"))
+
+ self.upload_env = upload_env
+ return self.upload_env
+
+ def _ensure_upload_path(self):
+ env = self._query_upload_env()
+ if "UPLOAD_PATH" in env and not os.path.exists(env["UPLOAD_PATH"]):
+ self.mkdir_p(env["UPLOAD_PATH"])
+
+ def get_gecko_l10n_revisions(self):
+ # Populate self.locales with Thunderbird's locales, and revisions
+ # from browser/locales/l10n-changesets.json
+ c = self.config
+ ignore_locales = c.get("ignore_locales", [])
+
+ dirs = self.query_abs_dirs()
+ locale_list = os.path.join(dirs["abs_src_dir"], c["locale_list"])
+ locales = self.parse_locales_file(locale_list)
+ locale_changesets_file = os.path.join(dirs["abs_src_dir"], c["locales_file"])
+ # parse_locales_file fills in self.l10n_revisions with changesets
+ self.parse_locales_file(locale_changesets_file)
+
+ for locale in ignore_locales:
+ if locale in locales:
+ self.debug("Ignoring locale %s." % locale)
+ locales.remove(locale)
+
+ self.locales = locales
+
+ # Actions {{{2
+ def clone_gecko_locales(self):
+ self.get_gecko_l10n_revisions()
+ self.pull_locale_source()
+
+ def clone_monorepo(self):
+ c = self.config
+ dirs = self.query_abs_dirs()
+
+ locales_file = os.path.join(dirs["abs_src_dir"], c["hg_comm_locales_file"])
+ locales_data = {}
+ if locales_file.endswith(".json"):
+ with open(locales_file) as fh:
+ locales_data = json.load(fh)
+ # would use en-US, but it's not in this file!
+ self.comm_l10n_revision = locales_data.get("en-GB", {}).get("revision")
+
+ if self.comm_l10n_revision:
+ self.mkdir_p(dirs["abs_checkout_dir"])
+ self.vcs_checkout(
+ dest=dirs["abs_comm_l10n_dir"],
+ repo=c["hg_comm_l10n_repo"],
+ branch=self.comm_l10n_revision,
+ vcs="hg",
+ )
+ else:
+ raise Exception(
+ f"Unable to find revision from comm-l10n repo using "
+ f"{c['hg_comm_locales_file']}."
+ )
+
+ def merge_repos(self):
+ dirs = self.query_abs_dirs()
+ if os.path.exists(dirs["abs_merged_dir"]):
+ self.rmtree(dirs["abs_merged_dir"])
+
+ file_registry = FileRegistry()
+
+ def add_to_registry(base_path, patterns):
+ finder = FileFinder(base_path)
+ for pattern in patterns:
+ for _lang in self.locales:
+ for _filepath, _fileobj in finder.find(pattern.format(lang=_lang)):
+ _filepath = os.path.join("l10n-central", _filepath)
+ file_registry.add(_filepath, _fileobj)
+
+ add_to_registry(dirs["abs_l10n_central_dir"], GECKO_STRINGS_PATTERNS)
+ add_to_registry(dirs["abs_comm_l10n_dir"], COMM_STRINGS_PATTERNS)
+
+ self.file_registry = file_registry
+
+ def pack_merged(self):
+ self._ensure_upload_path()
+ upload_path = self.config["upload_env"]["UPLOAD_PATH"]
+ archive_path = os.path.join(upload_path, self.config["upload_file"])
+
+ with open(archive_path, "wb") as f:
+ with ZstdCompressor().stream_writer(f) as z:
+ create_tar_from_files(z, dict(self.file_registry))
+
+ def gen_changesets(self):
+ changeset_data = {
+ "gecko_strings": self.gecko_locale_revisions,
+ "comm_strings": {
+ "repo": self.config["hg_comm_l10n_repo"],
+ "revision": self.comm_l10n_revision,
+ },
+ }
+ upload_path = self.config["upload_env"]["UPLOAD_PATH"]
+ changesets_file = os.path.join(upload_path, self.config["changesets_file"])
+ with open(changesets_file, "w") as f:
+ json.dump(changeset_data, f, sort_keys=True, indent=2)
+
+
+if __name__ == "__main__":
+ single_locale = CommMultiLocale()
+ single_locale.run_and_exit()
diff --git a/comm/taskcluster/scripts/source-test-clang-format.sh b/comm/taskcluster/scripts/source-test-clang-format.sh
new file mode 100755
index 0000000000..90f449a03b
--- /dev/null
+++ b/comm/taskcluster/scripts/source-test-clang-format.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+
+# Run the Firefox setup script
+source "$HOME/checkouts/gecko/taskcluster/scripts/misc/source-test-clang-setup.sh"
+
+# Append comm/.clang-format-ignore contents to $topsrcdir/.clang-format-ignore
+sed -e 's%^\([a-z]\)%comm/\1%' comm/.clang-format-ignore >> .clang-format-ignore
+
+# Update mozconfig file with Thunderbird build options
+cat <<EOT >> "$MOZCONFIG"
+ac_add_options --enable-project=comm/mail
+EOT
+
+# Run mach clang-format
+# shellcheck disable=SC2068
+./mach --log-no-times clang-format --output "$HOME/clang-format.json" --format json -p $@
+# shellcheck disable=SC2068
+./mach --log-no-times clang-format --output "$HOME/clang-format.diff" --format diff -p $@
+
+# Exit with an error code if clang-format.diff contains a proper diff.
+# Needed because mach clang-format will exit 0 regardless of outcome.
+# If no formatting is needed, clang-format.diff will have a single \n,
+# so check for a file size > 1 byte.
+DIFF_SIZE=$(stat -c %s "$HOME/clang-format.diff")
+if [[ "$DIFF_SIZE" -gt 1 ]]; then
+ echo "Exiting with error status. DIFF_SIZE is $DIFF_SIZE."
+ exit 1
+else
+ exit 0
+fi