diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-17 12:18:21 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-17 12:18:21 +0000 |
| commit | 4e8199b572f2035b7749cba276ece3a26630d23e (patch) | |
| tree | f09feeed6a0fe39d027b1908aa63ea6b35e4b631 /src/ci/docker | |
| parent | Adding upstream version 1.66.0+dfsg1. (diff) | |
| download | rustc-4e8199b572f2035b7749cba276ece3a26630d23e.tar.xz rustc-4e8199b572f2035b7749cba276ece3a26630d23e.zip | |
Adding upstream version 1.67.1+dfsg1.upstream/1.67.1+dfsg1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/ci/docker')
35 files changed, 1342 insertions, 67 deletions
diff --git a/src/ci/docker/host-aarch64/aarch64-gnu/Dockerfile b/src/ci/docker/host-aarch64/aarch64-gnu/Dockerfile index e54d0eafb..699938c37 100644 --- a/src/ci/docker/host-aarch64/aarch64-gnu/Dockerfile +++ b/src/ci/docker/host-aarch64/aarch64-gnu/Dockerfile @@ -14,7 +14,8 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins gdb \ libssl-dev \ pkg-config \ - xz-utils + xz-utils \ + && rm -rf /var/lib/apt/lists/* COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh diff --git a/src/ci/docker/host-x86_64/armhf-gnu/Dockerfile b/src/ci/docker/host-x86_64/armhf-gnu/Dockerfile index 69f88e495..57e63cd39 100644 --- a/src/ci/docker/host-x86_64/armhf-gnu/Dockerfile +++ b/src/ci/docker/host-x86_64/armhf-gnu/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get update -y && DEBIAN_FRONTEND=noninteractive apt-get install -y --no- curl \ file \ g++ \ - gcc-arm-linux-gnueabihf \ + g++-arm-linux-gnueabihf \ git \ libc6-dev \ libc6-dev-armhf-cross \ diff --git a/src/ci/docker/host-x86_64/disabled/dist-x86_64-haiku/Dockerfile b/src/ci/docker/host-x86_64/disabled/dist-x86_64-haiku/Dockerfile index 637b5fa22..5ddd3f180 100644 --- a/src/ci/docker/host-x86_64/disabled/dist-x86_64-haiku/Dockerfile +++ b/src/ci/docker/host-x86_64/disabled/dist-x86_64-haiku/Dockerfile @@ -47,6 +47,4 @@ ENV RUST_CONFIGURE_ARGS --disable-jemalloc \ --set=$TARGET.cc=x86_64-unknown-haiku-gcc \ --set=$TARGET.cxx=x86_64-unknown-haiku-g++ \ --set=$TARGET.llvm-config=/bin/llvm-config-haiku -ENV EXTERNAL_LLVM 1 - ENV SCRIPT python3 ../x.py dist --host=$HOST --target=$HOST diff --git a/src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile index cd86d9fb5..ea4a2a242 100644 --- a/src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile @@ -32,7 +32,8 @@ RUN yum upgrade -y && \ wget \ xz \ zlib-devel.i686 \ - zlib-devel.x86_64 + zlib-devel.x86_64 \ + && yum clean all RUN mkdir -p /rustroot/bin && ln -s /usr/bin/cmake3 /rustroot/bin/cmake diff --git a/src/ci/docker/host-x86_64/dist-powerpc64le-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-powerpc64le-linux/Dockerfile index 9a290edd5..6b7b32a8b 100644 --- a/src/ci/docker/host-x86_64/dist-powerpc64le-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-powerpc64le-linux/Dockerfile @@ -24,5 +24,5 @@ ENV \ ENV HOSTS=powerpc64le-unknown-linux-gnu -ENV RUST_CONFIGURE_ARGS --enable-extended --disable-docs +ENV RUST_CONFIGURE_ARGS --enable-extended --enable-profiler --disable-docs ENV SCRIPT python3 ../x.py dist --host $HOSTS --target $HOSTS diff --git a/src/ci/docker/host-x86_64/dist-riscv64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-riscv64-linux/Dockerfile index 88b8c7ea3..397f9538b 100644 --- a/src/ci/docker/host-x86_64/dist-riscv64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-riscv64-linux/Dockerfile @@ -27,5 +27,5 @@ ENV CC_riscv64gc_unknown_linux_gnu=riscv64-unknown-linux-gnu-gcc \ ENV HOSTS=riscv64gc-unknown-linux-gnu -ENV RUST_CONFIGURE_ARGS --enable-extended --disable-docs +ENV RUST_CONFIGURE_ARGS --enable-extended --enable-profiler --disable-docs ENV SCRIPT python3 ../x.py dist --target $HOSTS --host $HOSTS diff --git a/src/ci/docker/host-x86_64/dist-s390x-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-s390x-linux/Dockerfile index 7d77fdd30..43a449b3a 100644 --- a/src/ci/docker/host-x86_64/dist-s390x-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-s390x-linux/Dockerfile @@ -28,5 +28,5 @@ ENV \ ENV HOSTS=s390x-unknown-linux-gnu -ENV RUST_CONFIGURE_ARGS --enable-extended --enable-lld --disable-docs +ENV RUST_CONFIGURE_ARGS --enable-extended --enable-lld --enable-profiler --disable-docs ENV SCRIPT python3 ../x.py dist --host $HOSTS --target $HOSTS diff --git a/src/ci/docker/host-x86_64/dist-various-2/Dockerfile b/src/ci/docker/host-x86_64/dist-various-2/Dockerfile index 8250ec0c3..93ef7dfcb 100644 --- a/src/ci/docker/host-x86_64/dist-various-2/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-various-2/Dockerfile @@ -61,6 +61,12 @@ ENV \ AR_i686_unknown_freebsd=i686-unknown-freebsd12-ar \ CC_i686_unknown_freebsd=i686-unknown-freebsd12-clang \ CXX_i686_unknown_freebsd=i686-unknown-freebsd12-clang++ \ + CC_aarch64_unknown_uefi=clang-11 \ + CXX_aarch64_unknown_uefi=clang++-11 \ + CC_i686_unknown_uefi=clang-11 \ + CXX_i686_unknown_uefi=clang++-11 \ + CC_x86_64_unknown_uefi=clang-11 \ + CXX_x86_64_unknown_uefi=clang++-11 \ CC=gcc-8 \ CXX=g++-8 @@ -118,6 +124,9 @@ ENV TARGETS=$TARGETS,armv7-unknown-linux-gnueabi ENV TARGETS=$TARGETS,armv7-unknown-linux-musleabi ENV TARGETS=$TARGETS,i686-unknown-freebsd ENV TARGETS=$TARGETS,x86_64-unknown-none +ENV TARGETS=$TARGETS,aarch64-unknown-uefi +ENV TARGETS=$TARGETS,i686-unknown-uefi +ENV TARGETS=$TARGETS,x86_64-unknown-uefi # As per https://bugs.launchpad.net/ubuntu/+source/gcc-defaults/+bug/1300211 # we need asm in the search path for gcc-8 (for gnux32) but not in the search path of the @@ -129,6 +138,4 @@ ENV RUST_CONFIGURE_ARGS --enable-extended --enable-lld --disable-docs \ --set target.wasm32-wasi.wasi-root=/wasm32-wasi \ --musl-root-armv7=/musl-armv7 -ENV EXTERNAL_LLVM 1 - ENV SCRIPT python3 ../x.py dist --host='' --target $TARGETS diff --git a/src/ci/docker/host-x86_64/dist-various-2/build-wasi-toolchain.sh b/src/ci/docker/host-x86_64/dist-various-2/build-wasi-toolchain.sh index 36c94458d..67cee0148 100755 --- a/src/ci/docker/host-x86_64/dist-various-2/build-wasi-toolchain.sh +++ b/src/ci/docker/host-x86_64/dist-various-2/build-wasi-toolchain.sh @@ -2,15 +2,15 @@ set -ex -# Originally from https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.0/clang+llvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz -curl https://ci-mirrors.rust-lang.org/rustc/2022-05-10-clang%2Bllvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz | \ +# Originally from https://github.com/llvm/llvm-project/releases/download/llvmorg-15.0.6/clang+llvm-15.0.6-x86_64-linux-gnu-ubuntu-18.04.tar.xz +curl https://ci-mirrors.rust-lang.org/rustc/2022-12-06-clang%2Bllvm-15.0.6-x86_64-linux-gnu-ubuntu-18.04.tar.xz | \ tar xJf - -bin="$PWD/clang+llvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04/bin" +bin="$PWD/clang+llvm-15.0.6-x86_64-linux-gnu-ubuntu-18.04/bin" git clone https://github.com/WebAssembly/wasi-libc cd wasi-libc -git reset --hard 9886d3d6200fcc3726329966860fc058707406cd +git reset --hard 8b7148f69ae241a2749b3defe4606da8143b72e0 make -j$(nproc) \ CC="$bin/clang" \ NM="$bin/llvm-nm" \ diff --git a/src/ci/docker/host-x86_64/dist-x86_64-freebsd/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-freebsd/Dockerfile index f9b1fa895..377d4a9ce 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-freebsd/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-freebsd/Dockerfile @@ -15,7 +15,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ xz-utils \ wget \ libssl-dev \ - pkg-config + pkg-config \ + && rm -rf /var/lib/apt/lists/* COPY scripts/freebsd-toolchain.sh /tmp/ RUN /tmp/freebsd-toolchain.sh x86_64 diff --git a/src/ci/docker/host-x86_64/dist-x86_64-illumos/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-illumos/Dockerfile index c2e44ead5..4e46bdee5 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-illumos/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-illumos/Dockerfile @@ -11,7 +11,8 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends \ libgmp-dev \ libmpfr-dev \ - libmpc-dev + libmpc-dev \ + && rm -rf /var/lib/apt/lists/* COPY scripts/illumos-toolchain.sh /tmp/ diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index 423aba06c..6bdc88e18 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -32,7 +32,8 @@ RUN yum upgrade -y && \ wget \ xz \ zlib-devel.i686 \ - zlib-devel.x86_64 + zlib-devel.x86_64 \ + && yum clean all RUN mkdir -p /rustroot/bin && ln -s /usr/bin/cmake3 /rustroot/bin/cmake diff --git a/src/ci/docker/host-x86_64/dist-x86_64-musl/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-musl/Dockerfile index 51645a818..13eaf7fce 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-musl/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-musl/Dockerfile @@ -16,7 +16,8 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins gdb \ patch \ libssl-dev \ - pkg-config + pkg-config \ + && rm -rf /var/lib/apt/lists/* WORKDIR /build/ diff --git a/src/ci/docker/host-x86_64/dist-x86_64-netbsd/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-netbsd/Dockerfile index fed4be4c3..d03c36454 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-netbsd/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-netbsd/Dockerfile @@ -1,7 +1,8 @@ -FROM ubuntu:16.04 +FROM ubuntu:20.04 COPY scripts/cross-apt-packages.sh /scripts/ RUN sh /scripts/cross-apt-packages.sh +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y zlib1g-dev COPY host-x86_64/dist-x86_64-netbsd/build-netbsd-toolchain.sh /tmp/ RUN /tmp/build-netbsd-toolchain.sh @@ -9,9 +10,6 @@ RUN /tmp/build-netbsd-toolchain.sh COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh -COPY scripts/cmake.sh /scripts/ -RUN /scripts/cmake.sh - ENV PATH=$PATH:/x-tools/x86_64-unknown-netbsd/bin ENV \ @@ -21,6 +19,5 @@ ENV \ ENV HOSTS=x86_64-unknown-netbsd -ENV RUST_CONFIGURE_ARGS --enable-extended --disable-docs \ - --set llvm.allow-old-toolchain +ENV RUST_CONFIGURE_ARGS --enable-extended --disable-docs ENV SCRIPT python3 ../x.py dist --host $HOSTS --target $HOSTS diff --git a/src/ci/docker/host-x86_64/dist-x86_64-netbsd/build-netbsd-toolchain.sh b/src/ci/docker/host-x86_64/dist-x86_64-netbsd/build-netbsd-toolchain.sh index 5dfa47b4e..e0c008b76 100755 --- a/src/ci/docker/host-x86_64/dist-x86_64-netbsd/build-netbsd-toolchain.sh +++ b/src/ci/docker/host-x86_64/dist-x86_64-netbsd/build-netbsd-toolchain.sh @@ -25,19 +25,19 @@ cd netbsd mkdir -p /x-tools/x86_64-unknown-netbsd/sysroot -URL=https://ci-mirrors.rust-lang.org/rustc - -# Originally from ftp://ftp.netbsd.org/pub/NetBSD/NetBSD-$BSD/source/sets/*.tgz -curl $URL/2018-03-01-netbsd-src.tgz | tar xzf - -curl $URL/2018-03-01-netbsd-gnusrc.tgz | tar xzf - -curl $URL/2018-03-01-netbsd-sharesrc.tgz | tar xzf - -curl $URL/2018-03-01-netbsd-syssrc.tgz | tar xzf - - -# Originally from ftp://ftp.netbsd.org/pub/NetBSD/NetBSD-$BSD/amd64/binary/sets/*.tgz -curl $URL/2018-03-01-netbsd-base.tgz | \ - tar xzf - -C /x-tools/x86_64-unknown-netbsd/sysroot ./usr/include ./usr/lib ./lib -curl $URL/2018-03-01-netbsd-comp.tgz | \ - tar xzf - -C /x-tools/x86_64-unknown-netbsd/sysroot ./usr/include ./usr/lib +# URL=https://ci-mirrors.rust-lang.org/rustc + +SOURCE_URL=https://cdn.netbsd.org/pub/NetBSD/NetBSD-9.0/source/sets +curl $SOURCE_URL/src.tgz | tar xzf - +curl $SOURCE_URL/gnusrc.tgz | tar xzf - +curl $SOURCE_URL/sharesrc.tgz | tar xzf - +curl $SOURCE_URL/syssrc.tgz | tar xzf - + +BINARY_URL=https://cdn.netbsd.org/pub/NetBSD/NetBSD-9.0/amd64/binary/sets +curl $BINARY_URL/base.tar.xz | \ + tar xJf - -C /x-tools/x86_64-unknown-netbsd/sysroot ./usr/include ./usr/lib ./lib +curl $BINARY_URL/comp.tar.xz | \ + tar xJf - -C /x-tools/x86_64-unknown-netbsd/sysroot ./usr/include ./usr/lib cd usr/src diff --git a/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile b/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile index dd74726f8..e2b66c2cf 100644 --- a/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile +++ b/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile @@ -15,7 +15,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ gdb \ zlib1g-dev \ lib32z1-dev \ - xz-utils + xz-utils \ + && rm -rf /var/lib/apt/lists/* COPY scripts/sccache.sh /scripts/ diff --git a/src/ci/docker/host-x86_64/i686-gnu/Dockerfile b/src/ci/docker/host-x86_64/i686-gnu/Dockerfile index 0c36cfd66..cb6559707 100644 --- a/src/ci/docker/host-x86_64/i686-gnu/Dockerfile +++ b/src/ci/docker/host-x86_64/i686-gnu/Dockerfile @@ -15,7 +15,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ gdb \ zlib1g-dev \ lib32z1-dev \ - xz-utils + xz-utils \ + && rm -rf /var/lib/apt/lists/* COPY scripts/sccache.sh /scripts/ diff --git a/src/ci/docker/host-x86_64/mingw-check/Dockerfile b/src/ci/docker/host-x86_64/mingw-check/Dockerfile index 52a777615..40caa7c50 100644 --- a/src/ci/docker/host-x86_64/mingw-check/Dockerfile +++ b/src/ci/docker/host-x86_64/mingw-check/Dockerfile @@ -20,20 +20,20 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ xz-utils \ libssl-dev \ pkg-config \ - mingw-w64 + mingw-w64 \ + && rm -rf /var/lib/apt/lists/* RUN curl -sL https://nodejs.org/dist/v16.9.0/node-v16.9.0-linux-x64.tar.xz | tar -xJ ENV PATH="/node-v16.9.0-linux-x64/bin:${PATH}" # Install es-check # Pin its version to prevent unrelated CI failures due to future es-check versions. -RUN npm install es-check@6.1.1 -g -RUN npm install eslint@8.6.0 -g +RUN npm install es-check@6.1.1 eslint@8.6.0 -g COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh COPY host-x86_64/mingw-check/reuse-requirements.txt /tmp/ -RUN pip3 install --no-deps --require-hashes -r /tmp/reuse-requirements.txt +RUN pip3 install --no-deps --no-cache-dir --require-hashes -r /tmp/reuse-requirements.txt COPY host-x86_64/mingw-check/validate-toolstate.sh /scripts/ COPY host-x86_64/mingw-check/validate-error-codes.sh /scripts/ diff --git a/src/ci/docker/host-x86_64/test-various/Dockerfile b/src/ci/docker/host-x86_64/test-various/Dockerfile index b75e2f085..cf4451f8b 100644 --- a/src/ci/docker/host-x86_64/test-various/Dockerfile +++ b/src/ci/docker/host-x86_64/test-various/Dockerfile @@ -1,6 +1,7 @@ FROM ubuntu:20.04 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + clang-11 \ g++ \ make \ ninja-build \ @@ -16,11 +17,21 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins pkg-config \ xz-utils \ wget \ - patch + patch \ + ovmf \ + qemu-efi-aarch64 \ + qemu-system-arm \ + qemu-system-x86 \ + && rm -rf /var/lib/apt/lists/* RUN curl -sL https://nodejs.org/dist/v15.14.0/node-v15.14.0-linux-x64.tar.xz | \ tar -xJ +# Install 32-bit OVMF files for the i686-unknown-uefi test. This package +# is not available in ubuntu 20.04, so download a 22.04 package. +RUN curl -sL --output ovmf-ia32.deb http://mirrors.kernel.org/ubuntu/pool/universe/e/edk2/ovmf-ia32_2022.02-3_all.deb +RUN dpkg -i ovmf-ia32.deb && rm ovmf-ia32.deb + WORKDIR /build/ COPY scripts/musl-patch-configure.diff /build/ COPY scripts/musl-toolchain.sh /build/ @@ -64,4 +75,15 @@ ENV MUSL_TARGETS=x86_64-unknown-linux-musl \ CXX_x86_64_unknown_linux_musl=x86_64-linux-musl-g++ ENV MUSL_SCRIPT python3 /checkout/x.py --stage 2 test --host='' --target $MUSL_TARGETS -ENV SCRIPT $WASM_SCRIPT && $NVPTX_SCRIPT && $MUSL_SCRIPT +COPY host-x86_64/test-various/uefi_qemu_test /uefi_qemu_test +ENV UEFI_TARGETS=aarch64-unknown-uefi,i686-unknown-uefi,x86_64-unknown-uefi \ + CC_aarch64_unknown_uefi=clang-11 \ + CXX_aarch64_unknown_uefi=clang++-11 \ + CC_i686_unknown_uefi=clang-11 \ + CXX_i686_unknown_uefi=clang++-11 \ + CC_x86_64_unknown_uefi=clang-11 \ + CXX_x86_64_unknown_uefi=clang++-11 +ENV UEFI_SCRIPT python3 /checkout/x.py --stage 2 build --host='' --target $UEFI_TARGETS && \ + python3 -u /uefi_qemu_test/run.py + +ENV SCRIPT $WASM_SCRIPT && $NVPTX_SCRIPT && $MUSL_SCRIPT && $UEFI_SCRIPT diff --git a/src/ci/docker/host-x86_64/test-various/uefi_qemu_test/Cargo.toml b/src/ci/docker/host-x86_64/test-various/uefi_qemu_test/Cargo.toml new file mode 100644 index 000000000..fa8e5b3d0 --- /dev/null +++ b/src/ci/docker/host-x86_64/test-various/uefi_qemu_test/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "uefi_qemu_test" +version = "0.0.0" +edition = "2021" + +[workspace] + +[dependencies] +r-efi = "4.1.0" diff --git a/src/ci/docker/host-x86_64/test-various/uefi_qemu_test/run.py b/src/ci/docker/host-x86_64/test-various/uefi_qemu_test/run.py new file mode 100644 index 000000000..ffae7b0d4 --- /dev/null +++ b/src/ci/docker/host-x86_64/test-various/uefi_qemu_test/run.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 + +import os +import shutil +import subprocess +import sys +import tempfile + +from pathlib import Path + +TARGET_AARCH64 = 'aarch64-unknown-uefi' +TARGET_I686 = 'i686-unknown-uefi' +TARGET_X86_64 = 'x86_64-unknown-uefi' + +def run(*cmd, capture=False, check=True, env=None, timeout=None): + """Print and run a command, optionally capturing the output.""" + cmd = [str(p) for p in cmd] + print(' '.join(cmd)) + return subprocess.run(cmd, + capture_output=capture, + check=check, + env=env, + text=True, + timeout=timeout) + +def build_and_run(tmp_dir, target): + if target == TARGET_AARCH64: + boot_file_name = 'bootaa64.efi' + ovmf_dir = Path('/usr/share/AAVMF') + ovmf_code = 'AAVMF_CODE.fd' + ovmf_vars = 'AAVMF_VARS.fd' + qemu = 'qemu-system-aarch64' + machine = 'virt' + cpu = 'cortex-a72' + elif target == TARGET_I686: + boot_file_name = 'bootia32.efi' + ovmf_dir = Path('/usr/share/OVMF') + ovmf_code = 'OVMF32_CODE_4M.secboot.fd' + ovmf_vars = 'OVMF32_VARS_4M.fd' + # The i686 target intentionally uses 64-bit qemu; the important + # difference is that the OVMF code provides a 32-bit environment. + qemu = 'qemu-system-x86_64' + machine = 'q35' + cpu = 'qemu64' + elif target == TARGET_X86_64: + boot_file_name = 'bootx64.efi' + ovmf_dir = Path('/usr/share/OVMF') + ovmf_code = 'OVMF_CODE.fd' + ovmf_vars = 'OVMF_VARS.fd' + qemu = 'qemu-system-x86_64' + machine = 'q35' + cpu = 'qemu64' + else: + raise KeyError('invalid target') + + host_artifacts = Path('/checkout/obj/build/x86_64-unknown-linux-gnu') + stage0 = host_artifacts / 'stage0/bin' + stage2 = host_artifacts / 'stage2/bin' + + env = dict(os.environ) + env['PATH'] = '{}:{}:{}'.format(stage2, stage0, env['PATH']) + + # Copy the test create into `tmp_dir`. + test_crate = Path(tmp_dir) / 'uefi_qemu_test' + shutil.copytree('/uefi_qemu_test', test_crate) + + # Build the UEFI executable. + run('cargo', + 'build', + '--manifest-path', + test_crate / 'Cargo.toml', + '--target', + target, + env=env) + + # Create a mock EFI System Partition in a subdirectory. + esp = test_crate / 'esp' + boot = esp / 'efi/boot' + os.makedirs(boot, exist_ok=True) + + # Copy the executable into the ESP. + src_exe_path = test_crate / 'target' / target / 'debug/uefi_qemu_test.efi' + shutil.copy(src_exe_path, boot / boot_file_name) + print(src_exe_path, boot / boot_file_name) + + # Select the appropriate EDK2 build. + ovmf_code = ovmf_dir / ovmf_code + ovmf_vars = ovmf_dir / ovmf_vars + + # Make a writable copy of the vars file. aarch64 doesn't boot + # correctly with read-only vars. + ovmf_rw_vars = Path(tmp_dir) / 'vars.fd' + shutil.copy(ovmf_vars, ovmf_rw_vars) + + # Run the executable in QEMU and capture the output. + output = run(qemu, + '-machine', + machine, + '-cpu', + cpu, + '-display', + 'none', + '-serial', + 'stdio', + '-drive', + f'if=pflash,format=raw,readonly=on,file={ovmf_code}', + '-drive', + f'if=pflash,format=raw,readonly=off,file={ovmf_rw_vars}', + '-drive', + f'format=raw,file=fat:rw:{esp}', + capture=True, + # Ubuntu 20.04 (which is what the Dockerfile currently + # uses) provides QEMU 4.2.1, which segfaults on + # shutdown under some circumstances. That has been + # fixed in newer versions of QEMU, but for now just + # don't check the exit status. + check=False, + # Set a timeout to kill the VM in case something goes wrong. + timeout=60).stdout + + if 'Hello World!' in output: + print('VM produced expected output') + else: + print('unexpected VM output:') + print('---start---') + print(output) + print('---end---') + sys.exit(1) + + +def main(): + targets = [TARGET_AARCH64, TARGET_I686, TARGET_X86_64] + + for target in targets: + # Create a temporary directory so that we have a writeable + # workspace. + with tempfile.TemporaryDirectory() as tmp_dir: + build_and_run(tmp_dir, target) + + +if __name__ == "__main__": + main() diff --git a/src/ci/docker/host-x86_64/test-various/uefi_qemu_test/src/main.rs b/src/ci/docker/host-x86_64/test-various/uefi_qemu_test/src/main.rs new file mode 100644 index 000000000..2ec554c14 --- /dev/null +++ b/src/ci/docker/host-x86_64/test-various/uefi_qemu_test/src/main.rs @@ -0,0 +1,45 @@ +// Code is adapted from this hello world example: +// https://doc.rust-lang.org/nightly/rustc/platform-support/unknown-uefi.html + +#![no_main] +#![no_std] + +use core::{panic, ptr}; +use r_efi::efi::{Char16, Handle, Status, SystemTable, RESET_SHUTDOWN}; + +#[panic_handler] +fn panic_handler(_info: &panic::PanicInfo) -> ! { + loop {} +} + +#[export_name = "efi_main"] +pub extern "C" fn main(_h: Handle, st: *mut SystemTable) -> Status { + let s = [ + 0x0048u16, 0x0065u16, 0x006cu16, 0x006cu16, 0x006fu16, // "Hello" + 0x0020u16, // " " + 0x0057u16, 0x006fu16, 0x0072u16, 0x006cu16, 0x0064u16, // "World" + 0x0021u16, // "!" + 0x000au16, // "\n" + 0x0000u16, // NUL + ]; + + // Print "Hello World!". + let r = unsafe { ((*(*st).con_out).output_string)((*st).con_out, s.as_ptr() as *mut Char16) }; + if r.is_error() { + return r; + } + + // Shut down. + unsafe { + ((*((*st).runtime_services)).reset_system)( + RESET_SHUTDOWN, + Status::SUCCESS, + 0, + ptr::null_mut(), + ); + } + + // This should never be reached because `reset_system` should never + // return, so fail with an error if we get here. + Status::UNSUPPORTED +} diff --git a/src/ci/docker/host-x86_64/wasm32/Dockerfile b/src/ci/docker/host-x86_64/wasm32/Dockerfile index 9e37c2822..ef1fde1c3 100644 --- a/src/ci/docker/host-x86_64/wasm32/Dockerfile +++ b/src/ci/docker/host-x86_64/wasm32/Dockerfile @@ -13,7 +13,8 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins sudo \ gdb \ xz-utils \ - bzip2 + bzip2 \ + && rm -rf /var/lib/apt/lists/* COPY scripts/emscripten.sh /scripts/ RUN bash /scripts/emscripten.sh diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile index d55d5b56a..e08c4e1e8 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile @@ -19,7 +19,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ llvm-dev \ libfreetype6-dev \ libexpat1-dev \ - tidy + tidy \ + && rm -rf /var/lib/apt/lists/* COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile index 739045248..c2b002055 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile @@ -23,7 +23,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ pkg-config \ xz-utils \ lld \ - clang + clang \ + && rm -rf /var/lib/apt/lists/* COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile index 80a004501..7e640c49f 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile @@ -15,7 +15,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ gdb \ xz-utils \ libssl-dev \ - pkg-config + pkg-config \ + && rm -rf /var/lib/apt/lists/* COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile index 1289f116f..16976a942 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile @@ -21,7 +21,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ pkg-config \ zlib1g-dev \ xz-utils \ - nodejs + nodejs \ + && rm -rf /var/lib/apt/lists/* COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh @@ -29,7 +30,6 @@ RUN sh /scripts/sccache.sh # We are disabling CI LLVM since this builder is intentionally using a host # LLVM, rather than the typical src/llvm-project LLVM. ENV NO_DOWNLOAD_CI_LLVM 1 -ENV EXTERNAL_LLVM 1 # Using llvm-link-shared due to libffi issues -- see #34486 ENV RUST_CONFIGURE_ARGS \ diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile index 4b89a72ba..06f15bd12 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile @@ -24,15 +24,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ pkg-config \ zlib1g-dev \ xz-utils \ - nodejs - + nodejs \ + \ # Install powershell so we can test x.ps1 on Linux -RUN apt-get update && \ - apt-get install -y apt-transport-https software-properties-common && \ + apt-transport-https software-properties-common && \ curl -s "https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/packages-microsoft-prod.deb" > packages-microsoft-prod.deb && \ dpkg -i packages-microsoft-prod.deb && \ apt-get update && \ - apt-get install -y powershell + apt-get install -y powershell \ + && rm -rf /var/lib/apt/lists/* COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh @@ -40,7 +40,6 @@ RUN sh /scripts/sccache.sh # We are disabling CI LLVM since this builder is intentionally using a host # LLVM, rather than the typical src/llvm-project LLVM. ENV NO_DOWNLOAD_CI_LLVM 1 -ENV EXTERNAL_LLVM 1 # Using llvm-link-shared due to libffi issues -- see #34486 ENV RUST_CONFIGURE_ARGS \ diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-nopt/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-nopt/Dockerfile index 77510d7ac..9fdc78406 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-nopt/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-nopt/Dockerfile @@ -15,7 +15,8 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins gdb \ libssl-dev \ pkg-config \ - xz-utils + xz-utils \ + && rm -rf /var/lib/apt/lists/* COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile index 4350ca205..58c0c5db1 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile @@ -14,10 +14,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libssl-dev \ sudo \ xz-utils \ - tidy - + tidy \ + \ # Install dependencies for chromium browser -RUN apt-get install -y \ gconf-service \ libasound2 \ libatk1.0-0 \ @@ -56,7 +55,8 @@ RUN apt-get install -y \ libnss3 \ lsb-release \ xdg-utils \ - wget + wget \ + && rm -rf /var/lib/apt/lists/* COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version b/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version index cc96715b2..3f8dcd03d 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version @@ -1 +1 @@ -0.12.7
\ No newline at end of file +0.13.2
\ No newline at end of file diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh b/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh index 80a066cac..7dde63709 100755 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh @@ -27,6 +27,7 @@ python3 "$X_PY" test --stage 2 src/tools/rustfmt python3 "$X_PY" test --stage 2 src/tools/miri # We natively run this script on x86_64-unknown-linux-gnu and x86_64-pc-windows-msvc. # Also cover some other targets (on both of these hosts) via cross-testing. +export BOOTSTRAP_SKIP_TARGET_SANITY=1 # we don't need `cc` for these targets python3 "$X_PY" test --stage 2 src/tools/miri --target i686-pc-windows-msvc -#FIXME(https://github.com/rust-lang/rust/issues/103519): macOS testing is currently disabled -# python3 "$X_PY" test --stage 2 src/tools/miri --target aarch64-apple-darwin +python3 "$X_PY" test --stage 2 src/tools/miri --target aarch64-apple-darwin +unset BOOTSTRAP_SKIP_TARGET_SANITY diff --git a/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile index 88c182a4d..5b9581f72 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile @@ -15,7 +15,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ gdb \ libssl-dev \ pkg-config \ - xz-utils + xz-utils \ + && rm -rf /var/lib/apt/lists/* COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh diff --git a/src/ci/docker/scripts/freebsd-toolchain.sh b/src/ci/docker/scripts/freebsd-toolchain.sh index 4a4cac1b7..17cd456b9 100755 --- a/src/ci/docker/scripts/freebsd-toolchain.sh +++ b/src/ci/docker/scripts/freebsd-toolchain.sh @@ -53,7 +53,7 @@ files_to_extract=( for lib in c cxxrt gcc_s m thr util; do files_to_extract=("${files_to_extract[@]}" "./lib/lib${lib}.*" "./usr/lib/lib${lib}.*") done -for lib in c++ c_nonshared compiler_rt execinfo gcc pthread rt ssp_nonshared procstat devstat kvm; do +for lib in c++ c_nonshared compiler_rt execinfo gcc pthread rt ssp_nonshared procstat devstat kvm memstat; do files_to_extract=("${files_to_extract[@]}" "./usr/lib/lib${lib}.*") done diff --git a/src/ci/docker/scripts/fuchsia-test-runner.py b/src/ci/docker/scripts/fuchsia-test-runner.py new file mode 100755 index 000000000..a2708d169 --- /dev/null +++ b/src/ci/docker/scripts/fuchsia-test-runner.py @@ -0,0 +1,1041 @@ +#!/usr/bin/env python3 + +""" +The Rust toolchain test runner for Fuchsia. + +For instructions on running the compiler test suite, see +https://doc.rust-lang.org/stable/rustc/platform-support/fuchsia.html#aarch64-fuchsia-and-x86_64-fuchsia +""" + +import argparse +from dataclasses import dataclass +import glob +import hashlib +import json +import os +import platform +import re +import shutil +import signal +import subprocess +import sys +from typing import ClassVar, List + + +@dataclass +class TestEnvironment: + rust_dir: str + sdk_dir: str + target_arch: str + package_server_pid: int = None + emu_addr: str = None + libstd_name: str = None + libtest_name: str = None + verbose: bool = False + + @staticmethod + def tmp_dir(): + tmp_dir = os.environ.get("TEST_TOOLCHAIN_TMP_DIR") + if tmp_dir is not None: + return os.path.abspath(tmp_dir) + return os.path.join(os.path.dirname(__file__), "tmp~") + + @classmethod + def env_file_path(cls): + return os.path.join(cls.tmp_dir(), "test_env.json") + + @classmethod + def from_args(cls, args): + return cls( + os.path.abspath(args.rust), + os.path.abspath(args.sdk), + args.target_arch, + verbose=args.verbose, + ) + + @classmethod + def read_from_file(cls): + with open(cls.env_file_path(), encoding="utf-8") as f: + test_env = json.loads(f.read()) + return cls( + test_env["rust_dir"], + test_env["sdk_dir"], + test_env["target_arch"], + libstd_name=test_env["libstd_name"], + libtest_name=test_env["libtest_name"], + emu_addr=test_env["emu_addr"], + package_server_pid=test_env["package_server_pid"], + verbose=test_env["verbose"], + ) + + def image_name(self): + if self.target_arch == "x64": + return "qemu-x64" + if self.target_arch == "arm64": + return "qemu-arm64" + raise Exception(f"Unrecognized target architecture {self.target_arch}") + + def write_to_file(self): + with open(self.env_file_path(), "w", encoding="utf-8") as f: + f.write(json.dumps(self.__dict__)) + + def ssh_dir(self): + return os.path.join(self.tmp_dir(), "ssh") + + def ssh_keyfile_path(self): + return os.path.join(self.ssh_dir(), "fuchsia_ed25519") + + def ssh_authfile_path(self): + return os.path.join(self.ssh_dir(), "fuchsia_authorized_keys") + + def vdl_output_path(self): + return os.path.join(self.tmp_dir(), "vdl_output") + + def package_server_log_path(self): + return os.path.join(self.tmp_dir(), "package_server_log") + + def emulator_log_path(self): + return os.path.join(self.tmp_dir(), "emulator_log") + + def packages_dir(self): + return os.path.join(self.tmp_dir(), "packages") + + def output_dir(self): + return os.path.join(self.tmp_dir(), "output") + + TEST_REPO_NAME: ClassVar[str] = "rust-testing" + + def repo_dir(self): + return os.path.join(self.tmp_dir(), self.TEST_REPO_NAME) + + def rustlib_dir(self): + if self.target_arch == "x64": + return "x86_64-fuchsia" + if self.target_arch == "arm64": + return "aarch64-fuchsia" + raise Exception(f"Unrecognized target architecture {self.target_arch}") + + def libs_dir(self): + return os.path.join( + self.rust_dir, + "lib", + ) + + def rustlibs_dir(self): + return os.path.join( + self.libs_dir(), + "rustlib", + self.rustlib_dir(), + "lib", + ) + + def sdk_arch(self): + machine = platform.machine() + if machine == "x86_64": + return "x64" + if machine == "arm": + return "a64" + raise Exception(f"Unrecognized host architecture {machine}") + + def tool_path(self, tool): + return os.path.join(self.sdk_dir, "tools", self.sdk_arch(), tool) + + def host_arch_triple(self): + machine = platform.machine() + if machine == "x86_64": + return "x86_64-unknown-linux-gnu" + if machine == "arm": + return "aarch64-unknown-linux-gnu" + raise Exception(f"Unrecognized host architecture {machine}") + + def zxdb_script_path(self): + return os.path.join(self.tmp_dir(), "zxdb_script") + + def log_info(self, msg): + print(msg) + + def log_debug(self, msg): + if self.verbose: + print(msg) + + def subprocess_output(self): + if self.verbose: + return sys.stdout + return subprocess.DEVNULL + + def ffx_daemon_log_path(self): + return os.path.join(self.tmp_dir(), "ffx_daemon_log") + + def ffx_isolate_dir(self): + return os.path.join(self.tmp_dir(), "ffx_isolate") + + def ffx_home_dir(self): + return os.path.join(self.ffx_isolate_dir(), "user-home") + + def ffx_tmp_dir(self): + return os.path.join(self.ffx_isolate_dir(), "tmp") + + def ffx_log_dir(self): + return os.path.join(self.ffx_isolate_dir(), "log") + + def ffx_user_config_dir(self): + return os.path.join(self.ffx_xdg_config_home(), "Fuchsia", "ffx", "config") + + def ffx_user_config_path(self): + return os.path.join(self.ffx_user_config_dir(), "config.json") + + def ffx_xdg_config_home(self): + if platform.system() == "Darwin": + return os.path.join(self.ffx_home_dir(), "Library", "Preferences") + return os.path.join(self.ffx_home_dir(), ".local", "share") + + def ffx_ascendd_path(self): + return os.path.join(self.ffx_tmp_dir(), "ascendd") + + def start_ffx_isolation(self): + # Most of this is translated directly from ffx's isolate library + os.mkdir(self.ffx_isolate_dir()) + os.mkdir(self.ffx_home_dir()) + os.mkdir(self.ffx_tmp_dir()) + os.mkdir(self.ffx_log_dir()) + + fuchsia_dir = os.path.join(self.ffx_home_dir(), ".fuchsia") + os.mkdir(fuchsia_dir) + + fuchsia_debug_dir = os.path.join(fuchsia_dir, "debug") + os.mkdir(fuchsia_debug_dir) + + metrics_dir = os.path.join(fuchsia_dir, "metrics") + os.mkdir(metrics_dir) + + analytics_path = os.path.join(metrics_dir, "analytics-status") + with open(analytics_path, "w", encoding="utf-8") as analytics_file: + print("0", file=analytics_file) + + ffx_path = os.path.join(metrics_dir, "ffx") + with open(ffx_path, "w", encoding="utf-8") as ffx_file: + print("1", file=ffx_file) + + os.makedirs(self.ffx_user_config_dir()) + + with open( + self.ffx_user_config_path(), "w", encoding="utf-8" + ) as config_json_file: + user_config_for_test = { + "log": { + "enabled": True, + "dir": self.ffx_log_dir(), + }, + "overnet": { + "socket": self.ffx_ascendd_path(), + }, + "ssh": { + "pub": self.ssh_authfile_path(), + "priv": self.ssh_keyfile_path(), + }, + "test": { + "is_isolated": True, + "experimental_structured_output": True, + }, + } + print(json.dumps(user_config_for_test), file=config_json_file) + + ffx_env_path = os.path.join(self.ffx_user_config_dir(), ".ffx_env") + with open(ffx_env_path, "w", encoding="utf-8") as ffx_env_file: + ffx_env_config_for_test = { + "user": self.ffx_user_config_path(), + "build": None, + "global": None, + } + print(json.dumps(ffx_env_config_for_test), file=ffx_env_file) + + # Start ffx daemon + # We want this to be a long-running process that persists after the script finishes + # pylint: disable=consider-using-with + with open( + self.ffx_daemon_log_path(), "w", encoding="utf-8" + ) as ffx_daemon_log_file: + subprocess.Popen( + [ + self.tool_path("ffx"), + "--config", + self.ffx_user_config_path(), + "daemon", + "start", + ], + env=self.ffx_cmd_env(), + stdout=ffx_daemon_log_file, + stderr=ffx_daemon_log_file, + ) + + def ffx_cmd_env(self): + result = { + "HOME": self.ffx_home_dir(), + "XDG_CONFIG_HOME": self.ffx_xdg_config_home(), + "ASCENDD": self.ffx_ascendd_path(), + "FUCHSIA_SSH_KEY": self.ssh_keyfile_path(), + # We want to use our own specified temp directory + "TMP": self.tmp_dir(), + "TEMP": self.tmp_dir(), + "TMPDIR": self.tmp_dir(), + "TEMPDIR": self.tmp_dir(), + } + + return result + + def stop_ffx_isolation(self): + subprocess.check_call( + [ + self.tool_path("ffx"), + "--config", + self.ffx_user_config_path(), + "daemon", + "stop", + ], + env=self.ffx_cmd_env(), + stdout=self.subprocess_output(), + stderr=self.subprocess_output(), + ) + + def start(self): + """Sets up the testing environment and prepares to run tests. + + Args: + args: The command-line arguments to this command. + + During setup, this function will: + - Locate necessary shared libraries + - Create a new temp directory (this is where all temporary files are stored) + - Start an emulator + - Start an update server + - Create a new package repo and register it with the emulator + - Write test environment settings to a temporary file + """ + + # Initialize temp directory + if not os.path.exists(self.tmp_dir()): + os.mkdir(self.tmp_dir()) + elif len(os.listdir(self.tmp_dir())) != 0: + raise Exception(f"Temp directory is not clean (in {self.tmp_dir()})") + + os.mkdir(self.ssh_dir()) + os.mkdir(self.output_dir()) + + # Find libstd and libtest + libstd_paths = glob.glob(os.path.join(self.rustlibs_dir(), "libstd-*.so")) + libtest_paths = glob.glob(os.path.join(self.rustlibs_dir(), "libtest-*.so")) + + if not libstd_paths: + raise Exception(f"Failed to locate libstd (in {self.rustlibs_dir()})") + + if not libtest_paths: + raise Exception(f"Failed to locate libtest (in {self.rustlibs_dir()})") + + self.libstd_name = os.path.basename(libstd_paths[0]) + self.libtest_name = os.path.basename(libtest_paths[0]) + + # Generate SSH keys for the emulator to use + self.log_info("Generating SSH keys...") + subprocess.check_call( + [ + "ssh-keygen", + "-N", + "", + "-t", + "ed25519", + "-f", + self.ssh_keyfile_path(), + "-C", + "Generated by test_toolchain.py", + ], + stdout=self.subprocess_output(), + stderr=self.subprocess_output(), + ) + authfile_contents = subprocess.check_output( + [ + "ssh-keygen", + "-y", + "-f", + self.ssh_keyfile_path(), + ], + stderr=self.subprocess_output(), + ) + with open(self.ssh_authfile_path(), "wb") as authfile: + authfile.write(authfile_contents) + + # Start ffx isolation + self.log_info("Starting ffx isolation...") + self.start_ffx_isolation() + + # Start emulator (this will generate the vdl output) + self.log_info("Starting emulator...") + subprocess.check_call( + [ + self.tool_path("fvdl"), + "--sdk", + "start", + "--tuntap", + "--headless", + "--nointeractive", + "--ssh", + self.ssh_dir(), + "--vdl-output", + self.vdl_output_path(), + "--emulator-log", + self.emulator_log_path(), + "--image-name", + self.image_name(), + ], + stdout=self.subprocess_output(), + stderr=self.subprocess_output(), + ) + + # Parse vdl output for relevant information + with open(self.vdl_output_path(), encoding="utf-8") as f: + vdl_content = f.read() + matches = re.search( + r'network_address:\s+"\[([0-9a-f]{1,4}:(:[0-9a-f]{1,4}){4}%qemu)\]"', + vdl_content, + ) + self.emu_addr = matches.group(1) + + # Create new package repo + self.log_info("Creating package repo...") + subprocess.check_call( + [ + self.tool_path("pm"), + "newrepo", + "-repo", + self.repo_dir(), + ], + stdout=self.subprocess_output(), + stderr=self.subprocess_output(), + ) + + # Start package server + self.log_info("Starting package server...") + with open( + self.package_server_log_path(), "w", encoding="utf-8" + ) as package_server_log: + # We want this to be a long-running process that persists after the script finishes + # pylint: disable=consider-using-with + self.package_server_pid = subprocess.Popen( + [ + self.tool_path("pm"), + "serve", + "-vt", + "-repo", + self.repo_dir(), + "-l", + ":8084", + ], + stdout=package_server_log, + stderr=package_server_log, + ).pid + + # Register package server with emulator + self.log_info("Registering package server...") + ssh_client = subprocess.check_output( + [ + "ssh", + "-i", + self.ssh_keyfile_path(), + "-o", + "StrictHostKeyChecking=accept-new", + self.emu_addr, + "-f", + "echo $SSH_CLIENT", + ], + text=True, + ) + repo_addr = ssh_client.split()[0].replace("%", "%25") + repo_url = f"http://[{repo_addr}]:8084/config.json" + subprocess.check_call( + [ + "ssh", + "-i", + self.ssh_keyfile_path(), + "-o", + "StrictHostKeyChecking=accept-new", + self.emu_addr, + "-f", + f"pkgctl repo add url -f 1 -n {self.TEST_REPO_NAME} {repo_url}", + ], + stdout=self.subprocess_output(), + stderr=self.subprocess_output(), + ) + + # Write to file + self.write_to_file() + + self.log_info("Success! Your environment is ready to run tests.") + + # FIXME: shardify this + # `facet` statement required for TCP testing via + # protocol `fuchsia.posix.socket.Provider`. See + # https://fuchsia.dev/fuchsia-src/development/testing/components/test_runner_framework?hl=en#legacy_non-hermetic_tests + CML_TEMPLATE: ClassVar[ + str + ] = """ + {{ + program: {{ + runner: "elf_test_runner", + binary: "bin/{exe_name}", + forward_stderr_to: "log", + forward_stdout_to: "log", + environ: [{env_vars} + ] + }}, + capabilities: [ + {{ protocol: "fuchsia.test.Suite" }}, + ], + expose: [ + {{ + protocol: "fuchsia.test.Suite", + from: "self", + }}, + ], + use: [ + {{ storage: "data", path: "/data" }}, + {{ protocol: [ "fuchsia.process.Launcher" ] }}, + {{ protocol: [ "fuchsia.posix.socket.Provider" ] }} + ], + facets: {{ + "fuchsia.test": {{ type: "system" }}, + }}, + }} + """ + + MANIFEST_TEMPLATE = """ + meta/package={package_dir}/meta/package + meta/{package_name}.cm={package_dir}/meta/{package_name}.cm + bin/{exe_name}={bin_path} + lib/{libstd_name}={rust_dir}/lib/rustlib/{rustlib_dir}/lib/{libstd_name} + lib/{libtest_name}={rust_dir}/lib/rustlib/{rustlib_dir}/lib/{libtest_name} + lib/ld.so.1={sdk_dir}/arch/{target_arch}/sysroot/lib/libc.so + lib/libzircon.so={sdk_dir}/arch/{target_arch}/sysroot/lib/libzircon.so + lib/libfdio.so={sdk_dir}/arch/{target_arch}/lib/libfdio.so + """ + + TEST_ENV_VARS: ClassVar[List[str]] = [ + "TEST_EXEC_ENV", + "RUST_MIN_STACK", + "RUST_BACKTRACE", + "RUST_NEWRT", + "RUST_LOG", + "RUST_TEST_THREADS", + ] + + def run(self, args): + """Runs the requested test in the testing environment. + + Args: + args: The command-line arguments to this command. + Returns: + The return code of the test (0 for success, else failure). + + To run a test, this function will: + - Create, compile, archive, and publish a test package + - Run the test package on the emulator + - Forward the test's stdout and stderr as this script's stdout and stderr + """ + + bin_path = os.path.abspath(args.bin_path) + + # Build a unique, deterministic name for the test using the name of the + # binary and the last 6 hex digits of the hash of the full path + def path_checksum(path): + m = hashlib.sha256() + m.update(path.encode("utf-8")) + return m.hexdigest()[0:6] + + base_name = os.path.basename(os.path.dirname(args.bin_path)) + exe_name = base_name.lower().replace(".", "_") + package_name = f"{exe_name}_{path_checksum(bin_path)}" + + package_dir = os.path.join(self.packages_dir(), package_name) + cml_path = os.path.join(package_dir, "meta", f"{package_name}.cml") + cm_path = os.path.join(package_dir, "meta", f"{package_name}.cm") + manifest_path = os.path.join(package_dir, f"{package_name}.manifest") + far_path = os.path.join(package_dir, f"{package_name}-0.far") + + shared_libs = args.shared_libs[: args.n] + arguments = args.shared_libs[args.n :] + + test_output_dir = os.path.join(self.output_dir(), package_name) + + # Clean and create temporary output directory + if os.path.exists(test_output_dir): + shutil.rmtree(test_output_dir) + + os.mkdir(test_output_dir) + + # Open log file + log_path = os.path.join(test_output_dir, "log") + with open(log_path, "w", encoding="utf-8") as log_file: + + def log(msg): + print(msg, file=log_file) + log_file.flush() + + log(f"Bin path: {bin_path}") + + log("Setting up package...") + + # Set up package + subprocess.check_call( + [ + self.tool_path("pm"), + "-o", + package_dir, + "-n", + package_name, + "init", + ], + stdout=log_file, + stderr=log_file, + ) + + log("Writing CML...") + + # Write and compile CML + with open(cml_path, "w", encoding="utf-8") as cml: + # Collect environment variables + env_vars = "" + for var_name in self.TEST_ENV_VARS: + var_value = os.getenv(var_name) + if var_value is not None: + env_vars += f'\n "{var_name}={var_value}",' + + # Default to no backtrace for test suite + if os.getenv("RUST_BACKTRACE") == None: + env_vars += f'\n "RUST_BACKTRACE=0",' + + cml.write( + self.CML_TEMPLATE.format(env_vars=env_vars, exe_name=exe_name) + ) + + log("Compiling CML...") + + subprocess.check_call( + [ + self.tool_path("cmc"), + "compile", + cml_path, + "--includepath", + ".", + "--output", + cm_path, + ], + stdout=log_file, + stderr=log_file, + ) + + log("Writing manifest...") + + # Write, build, and archive manifest + with open(manifest_path, "w", encoding="utf-8") as manifest: + manifest.write( + self.MANIFEST_TEMPLATE.format( + bin_path=bin_path, + exe_name=exe_name, + package_dir=package_dir, + package_name=package_name, + rust_dir=self.rust_dir, + rustlib_dir=self.rustlib_dir(), + sdk_dir=self.sdk_dir, + libstd_name=self.libstd_name, + libtest_name=self.libtest_name, + target_arch=self.target_arch, + ) + ) + for shared_lib in shared_libs: + manifest.write(f"lib/{os.path.basename(shared_lib)}={shared_lib}\n") + + log("Compiling and archiving manifest...") + + subprocess.check_call( + [ + self.tool_path("pm"), + "-o", + package_dir, + "-m", + manifest_path, + "build", + ], + stdout=log_file, + stderr=log_file, + ) + subprocess.check_call( + [ + self.tool_path("pm"), + "-o", + package_dir, + "-m", + manifest_path, + "archive", + ], + stdout=log_file, + stderr=log_file, + ) + + log("Publishing package to repo...") + + # Publish package to repo + subprocess.check_call( + [ + self.tool_path("pm"), + "publish", + "-a", + "-repo", + self.repo_dir(), + "-f", + far_path, + ], + stdout=log_file, + stderr=log_file, + ) + + log("Running ffx test...") + + # Run test on emulator + subprocess.run( + [ + self.tool_path("ffx"), + "--config", + self.ffx_user_config_path(), + "test", + "run", + f"fuchsia-pkg://{self.TEST_REPO_NAME}/{package_name}#meta/{package_name}.cm", + "--min-severity-logs", + "TRACE", + "--output-directory", + test_output_dir, + "--", + ] + + arguments, + env=self.ffx_cmd_env(), + check=False, + stdout=log_file, + stderr=log_file, + ) + + log("Reporting test suite output...") + + # Read test suite output + run_summary_path = os.path.join(test_output_dir, "run_summary.json") + if os.path.exists(run_summary_path): + with open(run_summary_path, encoding="utf-8") as f: + run_summary = json.loads(f.read()) + + suite = run_summary["data"]["suites"][0] + case = suite["cases"][0] + + return_code = 0 if case["outcome"] == "PASSED" else 1 + + artifacts = case["artifacts"] + artifact_dir = case["artifact_dir"] + stdout_path = None + stderr_path = None + + for path, artifact in artifacts.items(): + artifact_path = os.path.join(test_output_dir, artifact_dir, path) + artifact_type = artifact["artifact_type"] + + if artifact_type == "STDERR": + stderr_path = artifact_path + elif artifact_type == "STDOUT": + stdout_path = artifact_path + + if stdout_path is not None and os.path.exists(stdout_path): + with open(stdout_path, encoding="utf-8") as f: + print(f.read(), file=sys.stdout, end="") + + if stderr_path is not None and os.path.exists(stderr_path): + with open(stderr_path, encoding="utf-8") as f: + print(f.read(), file=sys.stderr, end="") + else: + log("Failed to open test run summary") + return_code = 254 + + log("Done!") + + return return_code + + def stop(self): + """Shuts down and cleans up the testing environment. + + Args: + args: The command-line arguments to this command. + Returns: + The return code of the test (0 for success, else failure). + + During cleanup, this function will stop the emulator, package server, and + update server, then delete all temporary files. If an error is encountered + while stopping any running processes, the temporary files will not be deleted. + Passing --delete-tmp will force the process to delete the files anyway. + """ + + self.log_debug("Reporting logs...") + + # Print test log files + for test_dir in os.listdir(self.output_dir()): + log_path = os.path.join(self.output_dir(), test_dir, "log") + self.log_debug(f"\n---- Logs for test '{test_dir}' ----\n") + if os.path.exists(log_path): + with open(log_path, encoding="utf-8") as log: + self.log_debug(log.read()) + else: + self.log_debug("No logs found") + + # Print the emulator log + self.log_debug("\n---- Emulator logs ----\n") + if os.path.exists(self.emulator_log_path()): + with open(self.emulator_log_path(), encoding="utf-8") as log: + self.log_debug(log.read()) + else: + self.log_debug("No emulator logs found") + + # Print the package server log + self.log_debug("\n---- Package server log ----\n") + if os.path.exists(self.package_server_log_path()): + with open(self.package_server_log_path(), encoding="utf-8") as log: + self.log_debug(log.read()) + else: + self.log_debug("No package server log found") + + # Print the ffx daemon log + self.log_debug("\n---- ffx daemon log ----\n") + if os.path.exists(self.ffx_daemon_log_path()): + with open(self.ffx_daemon_log_path(), encoding="utf-8") as log: + self.log_debug(log.read()) + else: + self.log_debug("No ffx daemon log found") + + # Stop package server + self.log_info("Stopping package server...") + os.kill(self.package_server_pid, signal.SIGTERM) + + # Shut down the emulator + self.log_info("Stopping emulator...") + subprocess.check_call( + [ + self.tool_path("fvdl"), + "--sdk", + "kill", + "--launched-proto", + self.vdl_output_path(), + ], + stdout=self.subprocess_output(), + stderr=self.subprocess_output(), + ) + + # Stop ffx isolation + self.log_info("Stopping ffx isolation...") + self.stop_ffx_isolation() + + def delete_tmp(self): + # Remove temporary files + self.log_info("Deleting temporary files...") + shutil.rmtree(self.tmp_dir(), ignore_errors=True) + + def debug(self, args): + command = [ + self.tool_path("ffx"), + "--config", + self.ffx_user_config_path(), + "debug", + "connect", + "--", + "--build-id-dir", + os.path.join(self.sdk_dir, ".build-id"), + "--build-id-dir", + os.path.join(self.libs_dir(), ".build-id"), + ] + + # Add rust source if it's available + if args.rust_src is not None: + command += [ + "--build-dir", + args.rust_src, + ] + + # Add fuchsia source if it's available + if args.fuchsia_src is not None: + command += [ + "--build-dir", + os.path.join(args.fuchsia_src, "out", "default"), + ] + + # Load debug symbols for the test binary and automatically attach + if args.test is not None: + if args.rust_src is None: + raise Exception( + "A Rust source path is required with the `test` argument" + ) + + test_name = os.path.splitext(os.path.basename(args.test))[0] + + build_dir = os.path.join( + args.rust_src, + "fuchsia-build", + self.host_arch_triple(), + ) + test_dir = os.path.join( + build_dir, + "test", + os.path.dirname(args.test), + test_name, + ) + + with open(self.zxdb_script_path(), mode="w", encoding="utf-8") as f: + print(f"attach {test_name[:31]}", file=f) + + command += [ + "--symbol-path", + test_dir, + "-S", + self.zxdb_script_path(), + ] + + # Add any other zxdb arguments the user passed + if args.zxdb_args is not None: + command += args.zxdb_args + + # Connect to the running emulator with zxdb + subprocess.run(command, env=self.ffx_cmd_env(), check=False) + + +def start(args): + test_env = TestEnvironment.from_args(args) + test_env.start() + return 0 + + +def run(args): + test_env = TestEnvironment.read_from_file() + return test_env.run(args) + + +def stop(args): + test_env = TestEnvironment.read_from_file() + test_env.stop() + if not args.no_delete: + test_env.delete_tmp() + return 0 + + +def delete_tmp(args): + del args + test_env = TestEnvironment.read_from_file() + test_env.delete_tmp() + return 0 + + +def debug(args): + test_env = TestEnvironment.read_from_file() + test_env.debug(args) + return 0 + + +def main(): + parser = argparse.ArgumentParser() + + def print_help(args): + del args + parser.print_help() + return 0 + + parser.set_defaults(func=print_help) + + subparsers = parser.add_subparsers(help="valid sub-commands") + + start_parser = subparsers.add_parser( + "start", help="initializes the testing environment" + ) + start_parser.add_argument( + "--rust", + help="the directory of the installed Rust compiler for Fuchsia", + required=True, + ) + start_parser.add_argument( + "--sdk", + help="the directory of the fuchsia SDK", + required=True, + ) + start_parser.add_argument( + "--verbose", + help="prints more output from executed processes", + action="store_true", + ) + start_parser.add_argument( + "--target-arch", + help="the architecture of the image to test", + required=True, + ) + start_parser.set_defaults(func=start) + + run_parser = subparsers.add_parser( + "run", help="run a test in the testing environment" + ) + run_parser.add_argument( + "n", help="the number of shared libs passed along with the executable", type=int + ) + run_parser.add_argument("bin_path", help="path to the binary to run") + run_parser.add_argument( + "shared_libs", + help="the shared libs passed along with the binary", + nargs=argparse.REMAINDER, + ) + run_parser.set_defaults(func=run) + + stop_parser = subparsers.add_parser( + "stop", help="shuts down and cleans up the testing environment" + ) + stop_parser.add_argument( + "--no-delete", + default=False, + action="store_true", + help="don't delete temporary files after stopping", + ) + stop_parser.set_defaults(func=stop) + + delete_parser = subparsers.add_parser( + "delete-tmp", + help="deletes temporary files after the testing environment has been manually cleaned up", + ) + delete_parser.set_defaults(func=delete_tmp) + + debug_parser = subparsers.add_parser( + "debug", + help="connect to the active testing environment with zxdb", + ) + debug_parser.add_argument( + "--rust-src", + default=None, + help="the path to the Rust source being tested", + ) + debug_parser.add_argument( + "--fuchsia-src", + default=None, + help="the path to the Fuchsia source", + ) + debug_parser.add_argument( + "--test", + default=None, + help="the path to the test to debug (e.g. ui/box/new.rs)", + ) + debug_parser.add_argument( + "zxdb_args", + default=None, + nargs=argparse.REMAINDER, + help="any additional arguments to pass to zxdb", + ) + debug_parser.set_defaults(func=debug) + + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) |