diff options
Diffstat (limited to 'src/ci/docker')
18 files changed, 235 insertions, 192 deletions
diff --git a/src/ci/docker/host-x86_64/arm-android/Dockerfile b/src/ci/docker/host-x86_64/arm-android/Dockerfile index db11700af..abca06fb9 100644 --- a/src/ci/docker/host-x86_64/arm-android/Dockerfile +++ b/src/ci/docker/host-x86_64/arm-android/Dockerfile @@ -30,7 +30,7 @@ ENV PATH=$PATH:/android/sdk/platform-tools ENV TARGETS=arm-linux-androideabi -ENV RUST_CONFIGURE_ARGS --arm-linux-androideabi-ndk=/android/ndk/toolchains/llvm/prebuilt/linux-x86_64/ +ENV RUST_CONFIGURE_ARGS --android-ndk=/android/ndk/ ENV SCRIPT python3 ../x.py --stage 2 test --host='' --target $TARGETS diff --git a/src/ci/docker/host-x86_64/dist-android/Dockerfile b/src/ci/docker/host-x86_64/dist-android/Dockerfile index b09b6edb0..20b72b377 100644 --- a/src/ci/docker/host-x86_64/dist-android/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-android/Dockerfile @@ -19,12 +19,7 @@ ENV TARGETS=$TARGETS,x86_64-linux-android ENV RUST_CONFIGURE_ARGS \ --enable-extended \ --enable-profiler \ - --arm-linux-androideabi-ndk=/android/ndk/toolchains/llvm/prebuilt/linux-x86_64/ \ - --armv7-linux-androideabi-ndk=/android/ndk/toolchains/llvm/prebuilt/linux-x86_64/ \ - --thumbv7neon-linux-androideabi-ndk=/android/ndk/toolchains/llvm/prebuilt/linux-x86_64/ \ - --i686-linux-android-ndk=/android/ndk/toolchains/llvm/prebuilt/linux-x86_64/ \ - --aarch64-linux-android-ndk=/android/ndk/toolchains/llvm/prebuilt/linux-x86_64/ \ - --x86_64-linux-android-ndk=/android/ndk/toolchains/llvm/prebuilt/linux-x86_64/ \ + --android-ndk=/android/ndk/ \ --disable-docs ENV SCRIPT python3 ../x.py dist --host='' --target $TARGETS diff --git a/src/ci/docker/host-x86_64/dist-various-1/Dockerfile b/src/ci/docker/host-x86_64/dist-various-1/Dockerfile index 8f4ad0f4e..3372baed9 100644 --- a/src/ci/docker/host-x86_64/dist-various-1/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-various-1/Dockerfile @@ -29,8 +29,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ g++-arm-linux-gnueabi \ g++-arm-linux-gnueabihf \ g++-aarch64-linux-gnu \ - g++-mips64-linux-gnuabi64 \ - g++-mips64el-linux-gnuabi64 \ gcc-arm-none-eabi \ gcc-sparc64-linux-gnu \ libc6-dev-sparc64-cross \ @@ -48,12 +46,6 @@ WORKDIR /build COPY host-x86_64/dist-various-1/install-x86_64-redox.sh /build RUN ./install-x86_64-redox.sh -COPY host-x86_64/dist-various-1/install-mips-musl.sh /build -RUN ./install-mips-musl.sh - -COPY host-x86_64/dist-various-1/install-mipsel-musl.sh /build -RUN ./install-mipsel-musl.sh - COPY host-x86_64/dist-various-1/install-aarch64-none-elf.sh /build RUN ./install-aarch64-none-elf.sh @@ -76,32 +68,7 @@ RUN env \ env \ CC=arm-linux-gnueabihf-gcc CFLAGS="-march=armv7-a+fp" \ CXX=arm-linux-gnueabihf-g++ CXXFLAGS="-march=armv7-a+fp" \ - bash musl.sh armv7hf && \ - env \ - CC=mips-openwrt-linux-gcc \ - CXX=mips-openwrt-linux-g++ \ - bash musl.sh mips && \ - env \ - CC=mipsel-openwrt-linux-gcc \ - CXX=mipsel-openwrt-linux-g++ \ - bash musl.sh mipsel && \ - env \ - CC=mips64-linux-gnuabi64-gcc \ - CXX=mips64-linux-gnuabi64-g++ \ - bash musl.sh mips64 && \ - env \ - CC=mips64el-linux-gnuabi64-gcc \ - CXX=mips64el-linux-gnuabi64-g++ \ - bash musl.sh mips64el && \ - rm -rf /build/* - -# FIXME(mozilla/sccache#235) this shouldn't be necessary but is currently -# necessary to disambiguate the mips compiler with the mipsel compiler. We want -# to give these two wrapper scripts (currently identical ones) different hashes -# to ensure that sccache understands that they're different compilers. -RUN \ - echo "# a" >> /usr/local/mips-linux-musl/bin/mips-openwrt-linux-musl-wrapper.sh && \ - echo "# b" >> /usr/local/mipsel-linux-musl/bin/mipsel-openwrt-linux-musl-wrapper.sh + bash musl.sh armv7hf ENV RUN_MAKE_TARGETS=thumbv6m-none-eabi ENV RUN_MAKE_TARGETS=$RUN_MAKE_TARGETS,thumbv7m-none-eabi @@ -110,10 +77,6 @@ ENV RUN_MAKE_TARGETS=$RUN_MAKE_TARGETS,thumbv7em-none-eabihf ENV TARGETS=asmjs-unknown-emscripten ENV TARGETS=$TARGETS,wasm32-unknown-emscripten -ENV TARGETS=$TARGETS,mips-unknown-linux-musl -ENV TARGETS=$TARGETS,mipsel-unknown-linux-musl -ENV TARGETS=$TARGETS,mips64-unknown-linux-muslabi64 -ENV TARGETS=$TARGETS,mips64el-unknown-linux-muslabi64 ENV TARGETS=$TARGETS,arm-unknown-linux-musleabi ENV TARGETS=$TARGETS,arm-unknown-linux-musleabihf ENV TARGETS=$TARGETS,armv5te-unknown-linux-gnueabi @@ -149,10 +112,6 @@ ENV CFLAGS_armv5te_unknown_linux_musleabi="-march=armv5te -marm -mfloat-abi=soft CFLAGS_arm_unknown_linux_musleabi="-march=armv6 -marm" \ CFLAGS_arm_unknown_linux_musleabihf="-march=armv6 -marm -mfpu=vfp" \ CFLAGS_armv7_unknown_linux_musleabihf="-march=armv7-a+fp" \ - CC_mipsel_unknown_linux_musl=mipsel-openwrt-linux-gcc \ - CC_mips_unknown_linux_musl=mips-openwrt-linux-gcc \ - CC_mips64el_unknown_linux_muslabi64=mips64el-linux-gnuabi64-gcc \ - CC_mips64_unknown_linux_muslabi64=mips64-linux-gnuabi64-gcc \ CC_sparc64_unknown_linux_gnu=sparc64-linux-gnu-gcc \ CC_x86_64_unknown_redox=x86_64-unknown-redox-gcc \ CC_thumbv7neon_unknown_linux_gnueabihf=arm-linux-gnueabihf-gcc \ @@ -177,10 +136,6 @@ ENV RUST_CONFIGURE_ARGS \ --musl-root-arm=/musl-arm \ --musl-root-armhf=/musl-armhf \ --musl-root-armv7hf=/musl-armv7hf \ - --musl-root-mips=/musl-mips \ - --musl-root-mipsel=/musl-mipsel \ - --musl-root-mips64=/musl-mips64 \ - --musl-root-mips64el=/musl-mips64el \ --disable-docs ENV SCRIPT \ diff --git a/src/ci/docker/host-x86_64/dist-various-1/install-mips-musl.sh b/src/ci/docker/host-x86_64/dist-various-1/install-mips-musl.sh deleted file mode 100755 index abab18093..000000000 --- a/src/ci/docker/host-x86_64/dist-various-1/install-mips-musl.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -set -ex - -mkdir /usr/local/mips-linux-musl - -# originally from -# https://downloads.openwrt.org/snapshots/trunk/ar71xx/generic/ -# OpenWrt-Toolchain-ar71xx-generic_gcc-5.3.0_musl-1.1.16.Linux-x86_64.tar.bz2 -URL="https://ci-mirrors.rust-lang.org/rustc" -FILE="OpenWrt-Toolchain-ar71xx-generic_gcc-5.3.0_musl-1.1.16.Linux-x86_64.tar.bz2" -curl -L "$URL/$FILE" | tar xjf - -C /usr/local/mips-linux-musl --strip-components=2 - -for file in /usr/local/mips-linux-musl/bin/mips-openwrt-linux-*; do - ln -s $file /usr/local/bin/`basename $file` -done diff --git a/src/ci/docker/host-x86_64/dist-various-1/install-mipsel-musl.sh b/src/ci/docker/host-x86_64/dist-various-1/install-mipsel-musl.sh deleted file mode 100755 index 779acb2d8..000000000 --- a/src/ci/docker/host-x86_64/dist-various-1/install-mipsel-musl.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -set -ex - -mkdir /usr/local/mipsel-linux-musl - -# Note that this originally came from: -# https://downloads.openwrt.org/snapshots/trunk/malta/generic/ -# OpenWrt-Toolchain-malta-le_gcc-5.3.0_musl-1.1.15.Linux-x86_64.tar.bz2 -URL="https://ci-mirrors.rust-lang.org/rustc" -FILE="OpenWrt-Toolchain-malta-le_gcc-5.3.0_musl-1.1.15.Linux-x86_64.tar.bz2" -curl -L "$URL/$FILE" | tar xjf - -C /usr/local/mipsel-linux-musl --strip-components=2 - -for file in /usr/local/mipsel-linux-musl/bin/mipsel-openwrt-linux-*; do - ln -s $file /usr/local/bin/`basename $file` -done diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index 6f1b2a6a6..9a2fcb0ce 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -57,9 +57,9 @@ COPY host-x86_64/dist-x86_64-linux/build-clang.sh /tmp/ RUN ./build-clang.sh ENV CC=clang CXX=clang++ -# rustc-perf version from 2023-05-30 +# rustc-perf version from 2023-10-22 # Should also be changed in the opt-dist tool for other environments. -ENV PERF_COMMIT 8b2ac3042e1ff2c0074455a0a3618adef97156b1 +ENV PERF_COMMIT 4f313add609f43e928e98132358e8426ed3969ae RUN curl -LS -o perf.zip https://ci-mirrors.rust-lang.org/rustc/rustc-perf-$PERF_COMMIT.zip && \ unzip perf.zip && \ mv rustc-perf-$PERF_COMMIT rustc-perf && \ @@ -84,7 +84,8 @@ ENV RUST_CONFIGURE_ARGS \ --set llvm.ninja=false \ --set rust.jemalloc \ --set rust.use-lld=true \ - --set rust.lto=thin + --set rust.lto=thin \ + --set rust.codegen-units=1 ENV SCRIPT python3 ../x.py build --set rust.debug=true opt-dist && \ ./build/$HOSTS/stage0-tools-bin/opt-dist linux-ci -- python3 ../x.py dist \ diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh b/src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh index 02b023fe7..1d9568702 100755 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh @@ -4,7 +4,7 @@ set -ex source shared.sh -LLVM=llvmorg-17.0.0-rc3 +LLVM=llvmorg-17.0.4 mkdir llvm-project cd llvm-project diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh b/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh index 6da3f8922..e939a5d7e 100755 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh @@ -3,7 +3,8 @@ set -ex source shared.sh -GCC=8.5.0 +# Note: in the future when bumping to version 10.1.0, also take care of the sed block below. +GCC=9.5.0 curl https://ftp.gnu.org/gnu/gcc/gcc-$GCC/gcc-$GCC.tar.xz | xzcat | tar xf - cd gcc-$GCC @@ -22,15 +23,25 @@ cd gcc-$GCC # latter host is presented to `wget`! Therefore, we choose to download from the insecure HTTP server # instead here. # +# Note: in version 10.1.0, the URL used in `download_prerequisites` has changed from using FTP to +# using HTTP. When bumping to that gcc version, we can likely remove the sed replacement below, or +# the expression will need to be updated. That new URL is available at: +# https://github.com/gcc-mirror/gcc/blob/6e6e3f144a33ae504149dc992453b4f6dea12fdb/contrib/download_prerequisites#L35 +# sed -i'' 's|ftp://gcc\.gnu\.org/|https://gcc.gnu.org/|g' ./contrib/download_prerequisites ./contrib/download_prerequisites mkdir ../gcc-build cd ../gcc-build + +# '-fno-reorder-blocks-and-partition' is required to +# enable BOLT optimization of the C++ standard library, +# which is included in librustc_driver.so hide_output ../gcc-$GCC/configure \ --prefix=/rustroot \ --enable-languages=c,c++ \ - --disable-gnu-unique-object + --disable-gnu-unique-object \ + --enable-cxx-flags='-fno-reorder-blocks-and-partition' hide_output make -j$(nproc) hide_output make install ln -s gcc /rustroot/bin/cc diff --git a/src/ci/docker/host-x86_64/test-various/uefi_qemu_test/Cargo.lock b/src/ci/docker/host-x86_64/test-various/uefi_qemu_test/Cargo.lock new file mode 100644 index 000000000..e983edf20 --- /dev/null +++ b/src/ci/docker/host-x86_64/test-various/uefi_qemu_test/Cargo.lock @@ -0,0 +1,16 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "r-efi" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "575fc2d9b3da54adbdfaddf6eca48fec256d977c8630a1750b8991347d1ac911" + +[[package]] +name = "uefi_qemu_test" +version = "0.0.0" +dependencies = [ + "r-efi", +] diff --git a/src/ci/docker/host-x86_64/wasm32/Dockerfile b/src/ci/docker/host-x86_64/wasm32/Dockerfile deleted file mode 100644 index 0d0f1edd0..000000000 --- a/src/ci/docker/host-x86_64/wasm32/Dockerfile +++ /dev/null @@ -1,63 +0,0 @@ -FROM ubuntu:22.04 - -ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends \ - g++ \ - make \ - ninja-build \ - file \ - curl \ - ca-certificates \ - python3 \ - git \ - cmake \ - sudo \ - gdb \ - xz-utils \ - libssl-dev \ - bzip2 \ - && rm -rf /var/lib/apt/lists/* - -COPY scripts/emscripten.sh /scripts/ -RUN bash /scripts/emscripten.sh - -COPY scripts/sccache.sh /scripts/ -RUN sh /scripts/sccache.sh - -# emcc seems to need python to specifically be "python" and not "python3" -RUN ln `which python3` /usr/bin/python - -ENV PATH=$PATH:/emsdk-portable -ENV PATH=$PATH:/emsdk-portable/upstream/emscripten/ - -# Rust's build system requires NodeJS to be in the path, but the directory in -# which emsdk stores it contains the version number. This caused breakages in -# the past when emsdk bumped the node version causing the path to point to a -# missing directory. -# -# To avoid the problem this symlinks the latest NodeJs version available to -# "latest", and adds that to the path. -RUN ln -s /emsdk-portable/node/$(ls /emsdk-portable/node | sort -V | tail -n 1) \ - /emsdk-portable/node/latest -ENV PATH=$PATH:/emsdk-portable/node/latest/bin/ - -ENV BINARYEN_ROOT=/emsdk-portable/upstream/ -ENV EMSDK=/emsdk-portable -ENV EM_CONFIG=/emsdk-portable/.emscripten -ENV EM_CACHE=/emsdk-portable/upstream/emscripten/cache - -ENV TARGETS=wasm32-unknown-emscripten - -# Use -O1 optimizations in the link step to reduce time spent optimizing. -ENV EMCC_CFLAGS=-O1 - -COPY static/gitconfig /etc/gitconfig - -# Emscripten installation is user-specific -ENV NO_CHANGE_USER=1 -RUN chown 10719 -R /emsdk-portable/ - -# Exclude library/alloc due to OOM in benches. -# FIXME: Fix std tests -ENV SCRIPT python3 ../x.py test --stage 2 --host='' --target $TARGETS \ - --skip library/alloc --skip library/std diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile index 444e0275d..cefdcad76 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile @@ -24,6 +24,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ xz-utils \ nodejs \ mingw-w64 \ + libgccjit-12-dev \ && rm -rf /var/lib/apt/lists/* # Install powershell (universal package) so we can test x.ps1 on Linux @@ -34,6 +35,9 @@ RUN curl -sL "https://github.com/PowerShell/PowerShell/releases/download/v7.3.1/ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh +# Make `libgccjit.so` accessible to the linker. +RUN ln -s /usr/lib/gcc/x86_64-linux-gnu/12/libgccjit.so /usr/lib/x86_64-linux-gnu/libgccjit.so + # We are disabling CI LLVM since this builder is intentionally using a host # LLVM, rather than the typical src/llvm-project LLVM. ENV NO_DOWNLOAD_CI_LLVM 1 @@ -47,6 +51,7 @@ ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ --llvm-root=/usr/lib/llvm-15 \ --enable-llvm-link-shared \ + $USE_NEW_MANGLING \ --set rust.thin-lto-import-instr-limit=10 COPY host-x86_64/x86_64-gnu-llvm-15/script.sh /tmp/ diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh index 918b19612..2eb751ca3 100755 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh @@ -4,34 +4,46 @@ set -ex # Only run the stage 1 tests on merges, not on PR CI jobs. if [[ -z "${PR_CI_JOB}" ]]; then - ../x.py --stage 1 test --skip src/tools/tidy && \ - # Run the `mir-opt` tests again but this time for a 32-bit target. - # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have - # both 32-bit and 64-bit outputs updated by the PR author, before - # the PR is approved and tested for merging. - # It will also detect tests lacking `// EMIT_MIR_FOR_EACH_BIT_WIDTH`, - # despite having different output on 32-bit vs 64-bit targets. - ../x.py --stage 1 test tests/mir-opt \ - --host='' --target=i686-unknown-linux-gnu && \ - # Run `ui-fulldeps` in `--stage=1`, which actually uses the stage0 - # compiler, and is sensitive to the addition of new flags. - ../x.py --stage 1 test tests/ui-fulldeps + # When running gcc backend tests, we need to install `libgccjit` and to not run llvm codegen + # tests as it will fail them. + if [[ "${ENABLE_GCC_CODEGEN}" == "1" ]]; then + ../x.py --stage 1 test --skip src/tools/tidy --skip tests/codegen + else + ../x.py --stage 1 test --skip src/tools/tidy + fi + + # Run the `mir-opt` tests again but this time for a 32-bit target. + # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have + # both 32-bit and 64-bit outputs updated by the PR author, before + # the PR is approved and tested for merging. + # It will also detect tests lacking `// EMIT_MIR_FOR_EACH_BIT_WIDTH`, + # despite having different output on 32-bit vs 64-bit targets. + ../x.py --stage 1 test tests/mir-opt --host='' --target=i686-unknown-linux-gnu + + # Run `ui-fulldeps` in `--stage=1`, which actually uses the stage0 + # compiler, and is sensitive to the addition of new flags. + ../x.py --stage 1 test tests/ui-fulldeps fi +# When running gcc backend tests, we need to install `libgccjit` and to not run llvm codegen +# tests as it will fail them. # NOTE: intentionally uses all of `x.py`, `x`, and `x.ps1` to make sure they all work on Linux. -../x.py --stage 2 test --skip src/tools/tidy && \ - # Run the `mir-opt` tests again but this time for a 32-bit target. - # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have - # both 32-bit and 64-bit outputs updated by the PR author, before - # the PR is approved and tested for merging. - # It will also detect tests lacking `// EMIT_MIR_FOR_EACH_BIT_WIDTH`, - # despite having different output on 32-bit vs 64-bit targets. - ../x --stage 2 test tests/mir-opt \ - --host='' --target=i686-unknown-linux-gnu && \ - # Run the UI test suite again, but in `--pass=check` mode - # - # This is intended to make sure that both `--pass=check` continues to - # work. - # - ../x.ps1 --stage 2 test tests/ui --pass=check \ - --host='' --target=i686-unknown-linux-gnu +if [[ "${ENABLE_GCC_CODEGEN}" == "1" ]]; then + ../x.py --stage 2 test --skip src/tools/tidy --skip tests/codegen +else + ../x.py --stage 2 test --skip src/tools/tidy +fi + +# Run the `mir-opt` tests again but this time for a 32-bit target. +# This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have +# both 32-bit and 64-bit outputs updated by the PR author, before +# the PR is approved and tested for merging. +# It will also detect tests lacking `// EMIT_MIR_FOR_EACH_BIT_WIDTH`, +# despite having different output on 32-bit vs 64-bit targets. +../x --stage 2 test tests/mir-opt --host='' --target=i686-unknown-linux-gnu + +# Run the UI test suite again, but in `--pass=check` mode +# +# This is intended to make sure that both `--pass=check` continues to +# work. +../x.ps1 --stage 2 test tests/ui --pass=check --host='' --target=i686-unknown-linux-gnu diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-16/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-16/Dockerfile index 1e2b802e6..c177e7387 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-16/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-16/Dockerfile @@ -38,6 +38,10 @@ RUN sh /scripts/sccache.sh # LLVM, rather than the typical src/llvm-project LLVM. ENV NO_DOWNLOAD_CI_LLVM 1 +# This is not the latest LLVM version, so some components required by tests may +# be missing. +ENV IS_NOT_LATEST_LLVM 1 + # Using llvm-link-shared due to libffi issues -- see #34486 ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-17/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-17/Dockerfile new file mode 100644 index 000000000..76846f1fe --- /dev/null +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-17/Dockerfile @@ -0,0 +1,50 @@ +FROM ubuntu:23.10 + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + g++ \ + gcc-multilib \ + make \ + ninja-build \ + file \ + curl \ + ca-certificates \ + python3 \ + git \ + cmake \ + sudo \ + gdb \ + llvm-17-tools \ + llvm-17-dev \ + libedit-dev \ + libssl-dev \ + pkg-config \ + zlib1g-dev \ + xz-utils \ + nodejs \ + mingw-w64 \ + && rm -rf /var/lib/apt/lists/* + +# Install powershell (universal package) so we can test x.ps1 on Linux +RUN curl -sL "https://github.com/PowerShell/PowerShell/releases/download/v7.3.1/powershell_7.3.1-1.deb_amd64.deb" > powershell.deb && \ + dpkg -i powershell.deb && \ + rm -f powershell.deb + +COPY scripts/sccache.sh /scripts/ +RUN sh /scripts/sccache.sh + +# We are disabling CI LLVM since this builder is intentionally using a host +# LLVM, rather than the typical src/llvm-project LLVM. +ENV NO_DOWNLOAD_CI_LLVM 1 + +# Using llvm-link-shared due to libffi issues -- see #34486 +ENV RUST_CONFIGURE_ARGS \ + --build=x86_64-unknown-linux-gnu \ + --llvm-root=/usr/lib/llvm-17 \ + --enable-llvm-link-shared \ + --set rust.thin-lto-import-instr-limit=10 + +COPY host-x86_64/x86_64-gnu-llvm-15/script.sh /tmp/ + +ENV SCRIPT /tmp/script.sh diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile index 85f2f84a4..82385ea15 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile @@ -15,6 +15,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ sudo \ xz-utils \ tidy \ + libgccjit-12-dev \ \ # Install dependencies for chromium browser gconf-service \ @@ -61,6 +62,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh +# Make `libgccjit.so` accessible. +RUN ln -s /usr/lib/gcc/x86_64-linux-gnu/12/libgccjit.so /usr/lib/x86_64-linux-gnu/libgccjit.so +# Fix rustc_codegen_gcc lto issues. +ENV GCC_EXEC_PREFIX="/usr/lib/gcc/" + COPY host-x86_64/x86_64-gnu-tools/checktools.sh /tmp/ RUN curl -sL https://nodejs.org/dist/v14.20.0/node-v14.20.0-linux-x64.tar.xz | tar -xJ @@ -81,7 +87,10 @@ RUN npm install -g browser-ui-test@$(head -n 1 /tmp/browser-ui-test.version) --u ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ - --save-toolstates=/tmp/toolstate/toolstates.json + --save-toolstates=/tmp/toolstate/toolstates.json \ + --enable-new-symbol-mangling + +ENV HOST_TARGET x86_64-unknown-linux-gnu ENV SCRIPT /tmp/checktools.sh ../x.py && \ NODE_PATH=`npm root -g` python3 ../x.py test tests/rustdoc-gui --stage 2 \ diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh b/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh index 7dde63709..821a09feb 100755 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh @@ -1,4 +1,5 @@ #!/bin/sh +# ignore-tidy-linelength set -eu @@ -26,8 +27,30 @@ python3 "$X_PY" test --stage 2 src/tools/clippy python3 "$X_PY" test --stage 2 src/tools/rustfmt python3 "$X_PY" test --stage 2 src/tools/miri # We natively run this script on x86_64-unknown-linux-gnu and x86_64-pc-windows-msvc. -# Also cover some other targets (on both of these hosts) via cross-testing. +# Also cover some other targets via cross-testing, in particular all tier 1 targets. export BOOTSTRAP_SKIP_TARGET_SANITY=1 # we don't need `cc` for these targets -python3 "$X_PY" test --stage 2 src/tools/miri --target i686-pc-windows-msvc -python3 "$X_PY" test --stage 2 src/tools/miri --target aarch64-apple-darwin +case $HOST_TARGET in + x86_64-unknown-linux-gnu) + # Only this branch runs in PR CI. + # Fully test all main OSes, including a 32bit target. + python3 "$X_PY" test --stage 2 src/tools/miri --target x86_64-apple-darwin + python3 "$X_PY" test --stage 2 src/tools/miri --target i686-pc-windows-msvc + # Only run "pass" tests for the remaining targets, which is quite a bit faster. + python3 "$X_PY" test --stage 2 src/tools/miri --target x86_64-pc-windows-gnu --test-args pass + python3 "$X_PY" test --stage 2 src/tools/miri --target i686-unknown-linux-gnu --test-args pass + python3 "$X_PY" test --stage 2 src/tools/miri --target aarch64-unknown-linux-gnu --test-args pass + python3 "$X_PY" test --stage 2 src/tools/miri --target s390x-unknown-linux-gnu --test-args pass + ;; + x86_64-pc-windows-msvc) + # Strangely, Linux targets do not work here. cargo always says + # "error: cannot produce cdylib for ... as the target ... does not support these crate types". + # Only run "pass" tests, which is quite a bit faster. + python3 "$X_PY" test --stage 2 src/tools/miri --target aarch64-apple-darwin --test-args pass + python3 "$X_PY" test --stage 2 src/tools/miri --target i686-pc-windows-gnu --test-args pass + ;; + *) + echo "FATAL: unexpected host $HOST_TARGET" + exit 1 + ;; +esac unset BOOTSTRAP_SKIP_TARGET_SANITY diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index e9c155b13..cedbc0390 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -235,7 +235,7 @@ else args="$args --volume /tmp/toolstate:/tmp/toolstate" id=$(id -u) - if [[ "$id" != 0 && "$(docker -v)" =~ ^podman ]]; then + if [[ "$id" != 0 && "$(docker version)" =~ Podman ]]; then # Rootless podman creates a separate user namespace, where an inner # LOCAL_USER_ID will map to a different subuid range on the host. # The "keep-id" mode maps the current UID directly into the container. @@ -264,10 +264,27 @@ else BASE_COMMIT="" fi +SUMMARY_FILE=github-summary.md +touch $objdir/${SUMMARY_FILE} + +extra_env="" +if [ "$ENABLE_GCC_CODEGEN" = "1" ]; then + extra_env="$EXTRA_ENV --env ENABLE_GCC_CODEGEN=1" + # If `ENABLE_GCC_CODEGEN` is set and not empty, we add the `--enable-new-symbol-mangling` + # argument to `RUST_CONFIGURE_ARGS` and set the `GCC_EXEC_PREFIX` environment variable. + # `cg_gcc` doesn't support the legacy mangling so we need to enforce the new one + # if we run `cg_gcc` tests. + extra_env="$EXTRA_ENV --env USE_NEW_MANGLING=--enable-new-symbol-mangling" + # Fix rustc_codegen_gcc lto issues. + extra_env="$EXTRA_ENV --env GCC_EXEC_PREFIX=/usr/lib/gcc/" + echo "Setting extra environment values for docker: $extra_env" +fi + docker \ run \ --workdir /checkout/obj \ --env SRC=/checkout \ + $extra_env \ $args \ --env CARGO_HOME=/cargo \ --env DEPLOY \ @@ -275,6 +292,7 @@ docker \ --env CI \ --env GITHUB_ACTIONS \ --env GITHUB_REF \ + --env GITHUB_STEP_SUMMARY="/checkout/obj/${SUMMARY_FILE}" \ --env TOOLSTATE_REPO_ACCESS_TOKEN \ --env TOOLSTATE_REPO \ --env TOOLSTATE_PUBLISH \ @@ -284,11 +302,14 @@ docker \ --env DIST_TRY_BUILD \ --env PR_CI_JOB \ --env OBJDIR_ON_HOST="$objdir" \ + --env CODEGEN_BACKENDS \ --init \ --rm \ rust-ci \ $command +cat $objdir/${SUMMARY_FILE} >> "${GITHUB_STEP_SUMMARY}" + if [ -f /.dockerenv ]; then rm -rf $objdir docker cp checkout:/checkout/obj $objdir diff --git a/src/ci/docker/scripts/fuchsia-test-runner.py b/src/ci/docker/scripts/fuchsia-test-runner.py index f78d446c8..437b51641 100755 --- a/src/ci/docker/scripts/fuchsia-test-runner.py +++ b/src/ci/docker/scripts/fuchsia-test-runner.py @@ -12,6 +12,7 @@ from dataclasses import dataclass import fcntl import glob import hashlib +import io import json import os import platform @@ -276,27 +277,60 @@ class TestEnvironment: stderr=self.subprocess_output(), ) - # Start emulator - self.log_info("Starting emulator...") - product_bundle = "terminal.qemu-" + self.triple_to_arch(self.target) + # Look up the product bundle transfer manifest. + self.log_info("Looking up the product bundle transfer manifest...") + product_name = "minimal." + self.triple_to_arch(self.target) + fuchsia_version = "14.20230811.2.1" + + # FIXME: We should be able to replace this with the machine parsable + # `ffx --machine json product lookup ...` once F15 is released. + out = subprocess.check_output( + [ + ffx_path, + "product", + "lookup", + product_name, + fuchsia_version, + "--base-url", + "gs://fuchsia/development/" + fuchsia_version, + ], + env=ffx_env, + stderr=self.subprocess_output(), + ) + + self.log_debug(out) + + for line in io.BytesIO(out): + if line.startswith(b"gs://"): + transfer_manifest_url = line.rstrip() + break + else: + raise Exception("Unable to parse transfer manifest") + + # Download the product bundle. + product_bundle_dir = os.path.join(self.tmp_dir(), 'product-bundle') subprocess.check_call( [ ffx_path, - "product-bundle", - "get", - product_bundle, + "product", + "download", + transfer_manifest_url, + product_bundle_dir, + "--force", ], env=ffx_env, stdout=self.subprocess_output(), stderr=self.subprocess_output(), ) + + # Start emulator # FIXME: condition --accel hyper on target arch matching host arch subprocess.check_call( [ ffx_path, "emu", "start", - product_bundle, + product_bundle_dir, "--headless", "--log", self.emulator_log_path(), |