From 9835e2ae736235810b4ea1c162ca5e65c547e770 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sat, 18 May 2024 04:49:50 +0200 Subject: Merging upstream version 1.71.1+dfsg1. Signed-off-by: Daniel Baumann --- src/ci/docker/scripts/cross-apt-packages.sh | 1 + src/ci/docker/scripts/crosstool-ng-1.24.sh | 13 --- src/ci/docker/scripts/crosstool-ng-build.sh | 32 +++++++ src/ci/docker/scripts/crosstool-ng-git.sh | 17 ++++ src/ci/docker/scripts/crosstool-ng.sh | 13 ++- src/ci/docker/scripts/fuchsia-test-runner.py | 137 +++++++++------------------ src/ci/docker/scripts/musl-toolchain.sh | 18 ++-- src/ci/docker/scripts/musl.sh | 2 +- src/ci/docker/scripts/qemu-bare-bones-rcS | 2 +- 9 files changed, 113 insertions(+), 122 deletions(-) delete mode 100644 src/ci/docker/scripts/crosstool-ng-1.24.sh create mode 100755 src/ci/docker/scripts/crosstool-ng-build.sh create mode 100644 src/ci/docker/scripts/crosstool-ng-git.sh (limited to 'src/ci/docker/scripts') diff --git a/src/ci/docker/scripts/cross-apt-packages.sh b/src/ci/docker/scripts/cross-apt-packages.sh index 2f8bf1194..398362ca5 100644 --- a/src/ci/docker/scripts/cross-apt-packages.sh +++ b/src/ci/docker/scripts/cross-apt-packages.sh @@ -22,6 +22,7 @@ apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install patch \ pkg-config \ python3 \ + rsync \ sudo \ texinfo \ unzip \ diff --git a/src/ci/docker/scripts/crosstool-ng-1.24.sh b/src/ci/docker/scripts/crosstool-ng-1.24.sh deleted file mode 100644 index 3a40f6cdd..000000000 --- a/src/ci/docker/scripts/crosstool-ng-1.24.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh -set -ex - -# Mirrored from https://github.com/crosstool-ng/crosstool-ng/archive/crosstool-ng-1.24.0.tar.gz -url="https://ci-mirrors.rust-lang.org/rustc/crosstool-ng-1.24.0.tar.gz" -curl -Lf $url | tar xzf - -cd crosstool-ng-crosstool-ng-1.24.0 -./bootstrap -./configure --prefix=/usr/local -make -j$(nproc) -make install -cd .. -rm -rf crosstool-ng-crosstool-ng-1.24.0 diff --git a/src/ci/docker/scripts/crosstool-ng-build.sh b/src/ci/docker/scripts/crosstool-ng-build.sh new file mode 100755 index 000000000..c813d10eb --- /dev/null +++ b/src/ci/docker/scripts/crosstool-ng-build.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -ex + +if [ $UID -eq 0 ]; then + exec su rustbuild -c "$0" +fi + +hide_output() { + set +x + on_err=" +echo ERROR: An error was encountered with the build. +cat /tmp/build.log +exit 1 +" + trap "$on_err" ERR + bash -c "while true; do sleep 30; echo \$(date) - building ...; done" & + PING_LOOP_PID=$! + "$@" &> /tmp/build.log + rm /tmp/build.log + trap - ERR + kill $PING_LOOP_PID + set -x +} + +mkdir build +cd build +cp ../crosstool.defconfig .config +ct-ng olddefconfig +hide_output ct-ng build +cd .. +rm -rf build diff --git a/src/ci/docker/scripts/crosstool-ng-git.sh b/src/ci/docker/scripts/crosstool-ng-git.sh new file mode 100644 index 000000000..449cc476f --- /dev/null +++ b/src/ci/docker/scripts/crosstool-ng-git.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -ex + +URL=https://github.com/crosstool-ng/crosstool-ng +REV=943364711a650d9b9e84c1b42c91cc0265b6ab5c + +mkdir crosstool-ng +cd crosstool-ng +git init +git fetch --depth=1 ${URL} ${REV} +git reset --hard FETCH_HEAD +./bootstrap +./configure --prefix=/usr/local +make -j$(nproc) +make install +cd .. +rm -rf crosstool-ng diff --git a/src/ci/docker/scripts/crosstool-ng.sh b/src/ci/docker/scripts/crosstool-ng.sh index 1d0c28c8e..a28d7bde2 100644 --- a/src/ci/docker/scripts/crosstool-ng.sh +++ b/src/ci/docker/scripts/crosstool-ng.sh @@ -1,12 +1,19 @@ #!/bin/sh set -ex -url="https://github.com/crosstool-ng/crosstool-ng/archive/crosstool-ng-1.22.0.tar.gz" +CT_NG=1.25.0 + +url="https://github.com/crosstool-ng/crosstool-ng/archive/crosstool-ng-$CT_NG.tar.gz" curl -Lf $url | tar xzf - -cd crosstool-ng-crosstool-ng-1.22.0 +cd crosstool-ng-crosstool-ng-$CT_NG + +# https://github.com/crosstool-ng/crosstool-ng/issues/1832 +# "download source of zlib is invalid now" +sed -e "s|zlib.net/'|zlib.net/fossils'|" -i packages/zlib/package.desc + ./bootstrap ./configure --prefix=/usr/local make -j$(nproc) make install cd .. -rm -rf crosstool-ng-crosstool-ng-1.22.0 +rm -rf crosstool-ng-crosstool-ng-$CT_NG diff --git a/src/ci/docker/scripts/fuchsia-test-runner.py b/src/ci/docker/scripts/fuchsia-test-runner.py index e7d1d9781..ecef56f56 100755 --- a/src/ci/docker/scripts/fuchsia-test-runner.py +++ b/src/ci/docker/scripts/fuchsia-test-runner.py @@ -168,85 +168,17 @@ class TestEnvironment: def ffx_isolate_dir(self): return os.path.join(self.tmp_dir(), "ffx_isolate") - def ffx_home_dir(self): - return os.path.join(self.ffx_isolate_dir(), "user-home") + def home_dir(self): + return os.path.join(self.tmp_dir(), "user-home") - def ffx_tmp_dir(self): - return os.path.join(self.ffx_isolate_dir(), "tmp") - - def ffx_log_dir(self): - return os.path.join(self.ffx_isolate_dir(), "log") - - def ffx_user_config_dir(self): - return os.path.join(self.ffx_xdg_config_home(), "Fuchsia", "ffx", "config") - - def ffx_user_config_path(self): - return os.path.join(self.ffx_user_config_dir(), "config.json") - - def ffx_xdg_config_home(self): - if platform.system() == "Darwin": - return os.path.join(self.ffx_home_dir(), "Library", "Preferences") - return os.path.join(self.ffx_home_dir(), ".local", "share") - - def ffx_ascendd_path(self): - return os.path.join(self.ffx_tmp_dir(), "ascendd") def start_ffx_isolation(self): # Most of this is translated directly from ffx's isolate library os.mkdir(self.ffx_isolate_dir()) - os.mkdir(self.ffx_home_dir()) - os.mkdir(self.ffx_tmp_dir()) - os.mkdir(self.ffx_log_dir()) - - fuchsia_dir = os.path.join(self.ffx_home_dir(), ".fuchsia") - os.mkdir(fuchsia_dir) - - fuchsia_debug_dir = os.path.join(fuchsia_dir, "debug") - os.mkdir(fuchsia_debug_dir) - - metrics_dir = os.path.join(fuchsia_dir, "metrics") - os.mkdir(metrics_dir) + os.mkdir(self.home_dir()) - analytics_path = os.path.join(metrics_dir, "analytics-status") - with open(analytics_path, "w", encoding="utf-8") as analytics_file: - print("0", file=analytics_file) - - ffx_path = os.path.join(metrics_dir, "ffx") - with open(ffx_path, "w", encoding="utf-8") as ffx_file: - print("1", file=ffx_file) - - os.makedirs(self.ffx_user_config_dir()) - - with open( - self.ffx_user_config_path(), "w", encoding="utf-8" - ) as config_json_file: - user_config_for_test = { - "log": { - "enabled": True, - "dir": self.ffx_log_dir(), - }, - "overnet": { - "socket": self.ffx_ascendd_path(), - }, - "ssh": { - "pub": self.ssh_authfile_path(), - "priv": self.ssh_keyfile_path(), - }, - "test": { - "is_isolated": True, - "experimental_structured_output": True, - }, - } - print(json.dumps(user_config_for_test), file=config_json_file) - - ffx_env_path = os.path.join(self.ffx_user_config_dir(), ".ffx_env") - with open(ffx_env_path, "w", encoding="utf-8") as ffx_env_file: - ffx_env_config_for_test = { - "user": self.ffx_user_config_path(), - "build": None, - "global": None, - } - print(json.dumps(ffx_env_config_for_test), file=ffx_env_file) + ffx_path = self.tool_path("ffx") + ffx_env = self.ffx_cmd_env() # Start ffx daemon # We want this to be a long-running process that persists after the script finishes @@ -256,23 +188,54 @@ class TestEnvironment: ) as ffx_daemon_log_file: subprocess.Popen( [ - self.tool_path("ffx"), - "--config", - self.ffx_user_config_path(), + ffx_path, "daemon", "start", ], - env=self.ffx_cmd_env(), + env=ffx_env, stdout=ffx_daemon_log_file, stderr=ffx_daemon_log_file, ) + # Disable analytics + subprocess.check_call( + [ + ffx_path, + "config", + "analytics", + "disable", + ], + env=ffx_env, + stdout=self.subprocess_output(), + stderr=self.subprocess_output(), + ) + + # Set configs + configs = { + "log.enabled": "true", + "ssh.pub": self.ssh_authfile_path(), + "ssh.priv": self.ssh_keyfile_path(), + "test.is_isolated": "true", + "test.experimental_structured_output": "true", + } + for key, value in configs.items(): + subprocess.check_call( + [ + self.tool_path("ffx"), + "config", + "set", + key, + value, + ], + env=self.ffx_cmd_env(), + stdout=self.subprocess_output(), + stderr=self.subprocess_output(), + ) + def ffx_cmd_env(self): - result = { - "HOME": self.ffx_home_dir(), - "XDG_CONFIG_HOME": self.ffx_xdg_config_home(), - "ASCENDD": self.ffx_ascendd_path(), - "FUCHSIA_SSH_KEY": self.ssh_keyfile_path(), + return { + "HOME": self.home_dir(), + "FFX_ISOLATE_DIR": self.ffx_isolate_dir(), # We want to use our own specified temp directory "TMP": self.tmp_dir(), "TEMP": self.tmp_dir(), @@ -280,14 +243,10 @@ class TestEnvironment: "TEMPDIR": self.tmp_dir(), } - return result - def stop_ffx_isolation(self): subprocess.check_call( [ self.tool_path("ffx"), - "--config", - self.ffx_user_config_path(), "daemon", "stop", ], @@ -709,8 +668,6 @@ class TestEnvironment: subprocess.run( [ self.tool_path("ffx"), - "--config", - self.ffx_user_config_path(), "test", "run", f"fuchsia-pkg://{self.TEST_REPO_NAME}/{package_name}#meta/{package_name}.cm", @@ -849,8 +806,6 @@ class TestEnvironment: def debug(self, args): command = [ self.tool_path("ffx"), - "--config", - self.ffx_user_config_path(), "debug", "connect", "--", @@ -948,8 +903,6 @@ class TestEnvironment: subprocess.run( [ self.tool_path("ffx"), - "--config", - self.ffx_user_config_path(), "log", "--since", "now", diff --git a/src/ci/docker/scripts/musl-toolchain.sh b/src/ci/docker/scripts/musl-toolchain.sh index e358b8139..bc1b30e2d 100644 --- a/src/ci/docker/scripts/musl-toolchain.sh +++ b/src/ci/docker/scripts/musl-toolchain.sh @@ -4,7 +4,7 @@ # # Versions of the toolchain components are configurable in `musl-cross-make/Makefile` and # musl unlike GLIBC is forward compatible so upgrading it shouldn't break old distributions. -# Right now we have: Binutils 2.31.1, GCC 9.2.0, musl 1.1.24. +# Right now we have: Binutils 2.31.1, GCC 9.2.0, musl 1.2.3. # ignore-tidy-linelength @@ -32,6 +32,7 @@ TARGET=$ARCH-linux-musl # Don't depend on the mirrors of sabotage linux that musl-cross-make uses. LINUX_HEADERS_SITE=https://ci-mirrors.rust-lang.org/rustc/sabotage-linux-tarballs +LINUX_VER=headers-4.19.88 OUTPUT=/usr/local shift @@ -44,18 +45,11 @@ export CFLAGS="-fPIC -g1 $CFLAGS" git clone https://github.com/richfelker/musl-cross-make # -b v0.9.9 cd musl-cross-make -# A few commits ahead of v0.9.9 to include the cowpatch fix: -git checkout a54eb56f33f255dfca60be045f12a5cfaf5a72a9 +# A version that includes support for building musl 1.2.3 +git checkout fe915821b652a7fa37b34a596f47d8e20bc72338 -# Fix the cfi detection script in musl's configure so cfi is generated -# when debug info is asked for. This patch is derived from -# https://git.musl-libc.org/cgit/musl/commit/?id=c4d4028dde90562f631edf559fbc42d8ec1b29de. -# When we upgrade to a version that includes this commit, we can remove the patch. -mkdir patches/musl-1.1.24 -cp ../musl-patch-configure.diff patches/musl-1.1.24/0001-fix-cfi-detection.diff - -hide_output make -j$(nproc) TARGET=$TARGET MUSL_VER=1.1.24 LINUX_HEADERS_SITE=$LINUX_HEADERS_SITE -hide_output make install TARGET=$TARGET MUSL_VER=1.1.24 LINUX_HEADERS_SITE=$LINUX_HEADERS_SITE OUTPUT=$OUTPUT +hide_output make -j$(nproc) TARGET=$TARGET MUSL_VER=1.2.3 LINUX_HEADERS_SITE=$LINUX_HEADERS_SITE LINUX_VER=$LINUX_VER +hide_output make install TARGET=$TARGET MUSL_VER=1.2.3 LINUX_HEADERS_SITE=$LINUX_HEADERS_SITE LINUX_VER=$LINUX_VER OUTPUT=$OUTPUT cd - diff --git a/src/ci/docker/scripts/musl.sh b/src/ci/docker/scripts/musl.sh index 3e5dc4af0..ece8e6c15 100644 --- a/src/ci/docker/scripts/musl.sh +++ b/src/ci/docker/scripts/musl.sh @@ -25,7 +25,7 @@ shift # Apparently applying `-fPIC` everywhere allows them to link successfully. export CFLAGS="-fPIC $CFLAGS" -MUSL=musl-1.1.24 +MUSL=musl-1.2.3 # may have been downloaded in a previous run if [ ! -d $MUSL ]; then diff --git a/src/ci/docker/scripts/qemu-bare-bones-rcS b/src/ci/docker/scripts/qemu-bare-bones-rcS index 3c29bedc1..c5d807b2d 100644 --- a/src/ci/docker/scripts/qemu-bare-bones-rcS +++ b/src/ci/docker/scripts/qemu-bare-bones-rcS @@ -9,7 +9,7 @@ mount -t sysfs none /sys /addentropy < /addentropy cat /dev/urandom | head -n 2048 | /addentropy -# Set up IP that qemu expects. This confgures eth0 with the public IP that QEMU +# Set up IP that qemu expects. This configures eth0 with the public IP that QEMU # will communicate to as well as the loopback 127.0.0.1 address. ifconfig eth0 10.0.2.15 ifconfig lo up -- cgit v1.2.3