diff options
Diffstat (limited to 'src/ci')
23 files changed, 358 insertions, 227 deletions
diff --git a/src/ci/docker/host-x86_64/arm-android/Dockerfile b/src/ci/docker/host-x86_64/arm-android/Dockerfile index db11700af..abca06fb9 100644 --- a/src/ci/docker/host-x86_64/arm-android/Dockerfile +++ b/src/ci/docker/host-x86_64/arm-android/Dockerfile @@ -30,7 +30,7 @@ ENV PATH=$PATH:/android/sdk/platform-tools ENV TARGETS=arm-linux-androideabi -ENV RUST_CONFIGURE_ARGS --arm-linux-androideabi-ndk=/android/ndk/toolchains/llvm/prebuilt/linux-x86_64/ +ENV RUST_CONFIGURE_ARGS --android-ndk=/android/ndk/ ENV SCRIPT python3 ../x.py --stage 2 test --host='' --target $TARGETS diff --git a/src/ci/docker/host-x86_64/dist-android/Dockerfile b/src/ci/docker/host-x86_64/dist-android/Dockerfile index b09b6edb0..20b72b377 100644 --- a/src/ci/docker/host-x86_64/dist-android/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-android/Dockerfile @@ -19,12 +19,7 @@ ENV TARGETS=$TARGETS,x86_64-linux-android ENV RUST_CONFIGURE_ARGS \ --enable-extended \ --enable-profiler \ - --arm-linux-androideabi-ndk=/android/ndk/toolchains/llvm/prebuilt/linux-x86_64/ \ - --armv7-linux-androideabi-ndk=/android/ndk/toolchains/llvm/prebuilt/linux-x86_64/ \ - --thumbv7neon-linux-androideabi-ndk=/android/ndk/toolchains/llvm/prebuilt/linux-x86_64/ \ - --i686-linux-android-ndk=/android/ndk/toolchains/llvm/prebuilt/linux-x86_64/ \ - --aarch64-linux-android-ndk=/android/ndk/toolchains/llvm/prebuilt/linux-x86_64/ \ - --x86_64-linux-android-ndk=/android/ndk/toolchains/llvm/prebuilt/linux-x86_64/ \ + --android-ndk=/android/ndk/ \ --disable-docs ENV SCRIPT python3 ../x.py dist --host='' --target $TARGETS diff --git a/src/ci/docker/host-x86_64/dist-various-1/Dockerfile b/src/ci/docker/host-x86_64/dist-various-1/Dockerfile index 8f4ad0f4e..3372baed9 100644 --- a/src/ci/docker/host-x86_64/dist-various-1/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-various-1/Dockerfile @@ -29,8 +29,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ g++-arm-linux-gnueabi \ g++-arm-linux-gnueabihf \ g++-aarch64-linux-gnu \ - g++-mips64-linux-gnuabi64 \ - g++-mips64el-linux-gnuabi64 \ gcc-arm-none-eabi \ gcc-sparc64-linux-gnu \ libc6-dev-sparc64-cross \ @@ -48,12 +46,6 @@ WORKDIR /build COPY host-x86_64/dist-various-1/install-x86_64-redox.sh /build RUN ./install-x86_64-redox.sh -COPY host-x86_64/dist-various-1/install-mips-musl.sh /build -RUN ./install-mips-musl.sh - -COPY host-x86_64/dist-various-1/install-mipsel-musl.sh /build -RUN ./install-mipsel-musl.sh - COPY host-x86_64/dist-various-1/install-aarch64-none-elf.sh /build RUN ./install-aarch64-none-elf.sh @@ -76,32 +68,7 @@ RUN env \ env \ CC=arm-linux-gnueabihf-gcc CFLAGS="-march=armv7-a+fp" \ CXX=arm-linux-gnueabihf-g++ CXXFLAGS="-march=armv7-a+fp" \ - bash musl.sh armv7hf && \ - env \ - CC=mips-openwrt-linux-gcc \ - CXX=mips-openwrt-linux-g++ \ - bash musl.sh mips && \ - env \ - CC=mipsel-openwrt-linux-gcc \ - CXX=mipsel-openwrt-linux-g++ \ - bash musl.sh mipsel && \ - env \ - CC=mips64-linux-gnuabi64-gcc \ - CXX=mips64-linux-gnuabi64-g++ \ - bash musl.sh mips64 && \ - env \ - CC=mips64el-linux-gnuabi64-gcc \ - CXX=mips64el-linux-gnuabi64-g++ \ - bash musl.sh mips64el && \ - rm -rf /build/* - -# FIXME(mozilla/sccache#235) this shouldn't be necessary but is currently -# necessary to disambiguate the mips compiler with the mipsel compiler. We want -# to give these two wrapper scripts (currently identical ones) different hashes -# to ensure that sccache understands that they're different compilers. -RUN \ - echo "# a" >> /usr/local/mips-linux-musl/bin/mips-openwrt-linux-musl-wrapper.sh && \ - echo "# b" >> /usr/local/mipsel-linux-musl/bin/mipsel-openwrt-linux-musl-wrapper.sh + bash musl.sh armv7hf ENV RUN_MAKE_TARGETS=thumbv6m-none-eabi ENV RUN_MAKE_TARGETS=$RUN_MAKE_TARGETS,thumbv7m-none-eabi @@ -110,10 +77,6 @@ ENV RUN_MAKE_TARGETS=$RUN_MAKE_TARGETS,thumbv7em-none-eabihf ENV TARGETS=asmjs-unknown-emscripten ENV TARGETS=$TARGETS,wasm32-unknown-emscripten -ENV TARGETS=$TARGETS,mips-unknown-linux-musl -ENV TARGETS=$TARGETS,mipsel-unknown-linux-musl -ENV TARGETS=$TARGETS,mips64-unknown-linux-muslabi64 -ENV TARGETS=$TARGETS,mips64el-unknown-linux-muslabi64 ENV TARGETS=$TARGETS,arm-unknown-linux-musleabi ENV TARGETS=$TARGETS,arm-unknown-linux-musleabihf ENV TARGETS=$TARGETS,armv5te-unknown-linux-gnueabi @@ -149,10 +112,6 @@ ENV CFLAGS_armv5te_unknown_linux_musleabi="-march=armv5te -marm -mfloat-abi=soft CFLAGS_arm_unknown_linux_musleabi="-march=armv6 -marm" \ CFLAGS_arm_unknown_linux_musleabihf="-march=armv6 -marm -mfpu=vfp" \ CFLAGS_armv7_unknown_linux_musleabihf="-march=armv7-a+fp" \ - CC_mipsel_unknown_linux_musl=mipsel-openwrt-linux-gcc \ - CC_mips_unknown_linux_musl=mips-openwrt-linux-gcc \ - CC_mips64el_unknown_linux_muslabi64=mips64el-linux-gnuabi64-gcc \ - CC_mips64_unknown_linux_muslabi64=mips64-linux-gnuabi64-gcc \ CC_sparc64_unknown_linux_gnu=sparc64-linux-gnu-gcc \ CC_x86_64_unknown_redox=x86_64-unknown-redox-gcc \ CC_thumbv7neon_unknown_linux_gnueabihf=arm-linux-gnueabihf-gcc \ @@ -177,10 +136,6 @@ ENV RUST_CONFIGURE_ARGS \ --musl-root-arm=/musl-arm \ --musl-root-armhf=/musl-armhf \ --musl-root-armv7hf=/musl-armv7hf \ - --musl-root-mips=/musl-mips \ - --musl-root-mipsel=/musl-mipsel \ - --musl-root-mips64=/musl-mips64 \ - --musl-root-mips64el=/musl-mips64el \ --disable-docs ENV SCRIPT \ diff --git a/src/ci/docker/host-x86_64/dist-various-1/install-mips-musl.sh b/src/ci/docker/host-x86_64/dist-various-1/install-mips-musl.sh deleted file mode 100755 index abab18093..000000000 --- a/src/ci/docker/host-x86_64/dist-various-1/install-mips-musl.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -set -ex - -mkdir /usr/local/mips-linux-musl - -# originally from -# https://downloads.openwrt.org/snapshots/trunk/ar71xx/generic/ -# OpenWrt-Toolchain-ar71xx-generic_gcc-5.3.0_musl-1.1.16.Linux-x86_64.tar.bz2 -URL="https://ci-mirrors.rust-lang.org/rustc" -FILE="OpenWrt-Toolchain-ar71xx-generic_gcc-5.3.0_musl-1.1.16.Linux-x86_64.tar.bz2" -curl -L "$URL/$FILE" | tar xjf - -C /usr/local/mips-linux-musl --strip-components=2 - -for file in /usr/local/mips-linux-musl/bin/mips-openwrt-linux-*; do - ln -s $file /usr/local/bin/`basename $file` -done diff --git a/src/ci/docker/host-x86_64/dist-various-1/install-mipsel-musl.sh b/src/ci/docker/host-x86_64/dist-various-1/install-mipsel-musl.sh deleted file mode 100755 index 779acb2d8..000000000 --- a/src/ci/docker/host-x86_64/dist-various-1/install-mipsel-musl.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -set -ex - -mkdir /usr/local/mipsel-linux-musl - -# Note that this originally came from: -# https://downloads.openwrt.org/snapshots/trunk/malta/generic/ -# OpenWrt-Toolchain-malta-le_gcc-5.3.0_musl-1.1.15.Linux-x86_64.tar.bz2 -URL="https://ci-mirrors.rust-lang.org/rustc" -FILE="OpenWrt-Toolchain-malta-le_gcc-5.3.0_musl-1.1.15.Linux-x86_64.tar.bz2" -curl -L "$URL/$FILE" | tar xjf - -C /usr/local/mipsel-linux-musl --strip-components=2 - -for file in /usr/local/mipsel-linux-musl/bin/mipsel-openwrt-linux-*; do - ln -s $file /usr/local/bin/`basename $file` -done diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index 6f1b2a6a6..9a2fcb0ce 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -57,9 +57,9 @@ COPY host-x86_64/dist-x86_64-linux/build-clang.sh /tmp/ RUN ./build-clang.sh ENV CC=clang CXX=clang++ -# rustc-perf version from 2023-05-30 +# rustc-perf version from 2023-10-22 # Should also be changed in the opt-dist tool for other environments. -ENV PERF_COMMIT 8b2ac3042e1ff2c0074455a0a3618adef97156b1 +ENV PERF_COMMIT 4f313add609f43e928e98132358e8426ed3969ae RUN curl -LS -o perf.zip https://ci-mirrors.rust-lang.org/rustc/rustc-perf-$PERF_COMMIT.zip && \ unzip perf.zip && \ mv rustc-perf-$PERF_COMMIT rustc-perf && \ @@ -84,7 +84,8 @@ ENV RUST_CONFIGURE_ARGS \ --set llvm.ninja=false \ --set rust.jemalloc \ --set rust.use-lld=true \ - --set rust.lto=thin + --set rust.lto=thin \ + --set rust.codegen-units=1 ENV SCRIPT python3 ../x.py build --set rust.debug=true opt-dist && \ ./build/$HOSTS/stage0-tools-bin/opt-dist linux-ci -- python3 ../x.py dist \ diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh b/src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh index 02b023fe7..1d9568702 100755 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh @@ -4,7 +4,7 @@ set -ex source shared.sh -LLVM=llvmorg-17.0.0-rc3 +LLVM=llvmorg-17.0.4 mkdir llvm-project cd llvm-project diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh b/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh index 6da3f8922..e939a5d7e 100755 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh @@ -3,7 +3,8 @@ set -ex source shared.sh -GCC=8.5.0 +# Note: in the future when bumping to version 10.1.0, also take care of the sed block below. +GCC=9.5.0 curl https://ftp.gnu.org/gnu/gcc/gcc-$GCC/gcc-$GCC.tar.xz | xzcat | tar xf - cd gcc-$GCC @@ -22,15 +23,25 @@ cd gcc-$GCC # latter host is presented to `wget`! Therefore, we choose to download from the insecure HTTP server # instead here. # +# Note: in version 10.1.0, the URL used in `download_prerequisites` has changed from using FTP to +# using HTTP. When bumping to that gcc version, we can likely remove the sed replacement below, or +# the expression will need to be updated. That new URL is available at: +# https://github.com/gcc-mirror/gcc/blob/6e6e3f144a33ae504149dc992453b4f6dea12fdb/contrib/download_prerequisites#L35 +# sed -i'' 's|ftp://gcc\.gnu\.org/|https://gcc.gnu.org/|g' ./contrib/download_prerequisites ./contrib/download_prerequisites mkdir ../gcc-build cd ../gcc-build + +# '-fno-reorder-blocks-and-partition' is required to +# enable BOLT optimization of the C++ standard library, +# which is included in librustc_driver.so hide_output ../gcc-$GCC/configure \ --prefix=/rustroot \ --enable-languages=c,c++ \ - --disable-gnu-unique-object + --disable-gnu-unique-object \ + --enable-cxx-flags='-fno-reorder-blocks-and-partition' hide_output make -j$(nproc) hide_output make install ln -s gcc /rustroot/bin/cc diff --git a/src/ci/docker/host-x86_64/test-various/uefi_qemu_test/Cargo.lock b/src/ci/docker/host-x86_64/test-various/uefi_qemu_test/Cargo.lock new file mode 100644 index 000000000..e983edf20 --- /dev/null +++ b/src/ci/docker/host-x86_64/test-various/uefi_qemu_test/Cargo.lock @@ -0,0 +1,16 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "r-efi" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "575fc2d9b3da54adbdfaddf6eca48fec256d977c8630a1750b8991347d1ac911" + +[[package]] +name = "uefi_qemu_test" +version = "0.0.0" +dependencies = [ + "r-efi", +] diff --git a/src/ci/docker/host-x86_64/wasm32/Dockerfile b/src/ci/docker/host-x86_64/wasm32/Dockerfile deleted file mode 100644 index 0d0f1edd0..000000000 --- a/src/ci/docker/host-x86_64/wasm32/Dockerfile +++ /dev/null @@ -1,63 +0,0 @@ -FROM ubuntu:22.04 - -ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends \ - g++ \ - make \ - ninja-build \ - file \ - curl \ - ca-certificates \ - python3 \ - git \ - cmake \ - sudo \ - gdb \ - xz-utils \ - libssl-dev \ - bzip2 \ - && rm -rf /var/lib/apt/lists/* - -COPY scripts/emscripten.sh /scripts/ -RUN bash /scripts/emscripten.sh - -COPY scripts/sccache.sh /scripts/ -RUN sh /scripts/sccache.sh - -# emcc seems to need python to specifically be "python" and not "python3" -RUN ln `which python3` /usr/bin/python - -ENV PATH=$PATH:/emsdk-portable -ENV PATH=$PATH:/emsdk-portable/upstream/emscripten/ - -# Rust's build system requires NodeJS to be in the path, but the directory in -# which emsdk stores it contains the version number. This caused breakages in -# the past when emsdk bumped the node version causing the path to point to a -# missing directory. -# -# To avoid the problem this symlinks the latest NodeJs version available to -# "latest", and adds that to the path. -RUN ln -s /emsdk-portable/node/$(ls /emsdk-portable/node | sort -V | tail -n 1) \ - /emsdk-portable/node/latest -ENV PATH=$PATH:/emsdk-portable/node/latest/bin/ - -ENV BINARYEN_ROOT=/emsdk-portable/upstream/ -ENV EMSDK=/emsdk-portable -ENV EM_CONFIG=/emsdk-portable/.emscripten -ENV EM_CACHE=/emsdk-portable/upstream/emscripten/cache - -ENV TARGETS=wasm32-unknown-emscripten - -# Use -O1 optimizations in the link step to reduce time spent optimizing. -ENV EMCC_CFLAGS=-O1 - -COPY static/gitconfig /etc/gitconfig - -# Emscripten installation is user-specific -ENV NO_CHANGE_USER=1 -RUN chown 10719 -R /emsdk-portable/ - -# Exclude library/alloc due to OOM in benches. -# FIXME: Fix std tests -ENV SCRIPT python3 ../x.py test --stage 2 --host='' --target $TARGETS \ - --skip library/alloc --skip library/std diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile index 444e0275d..cefdcad76 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile @@ -24,6 +24,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ xz-utils \ nodejs \ mingw-w64 \ + libgccjit-12-dev \ && rm -rf /var/lib/apt/lists/* # Install powershell (universal package) so we can test x.ps1 on Linux @@ -34,6 +35,9 @@ RUN curl -sL "https://github.com/PowerShell/PowerShell/releases/download/v7.3.1/ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh +# Make `libgccjit.so` accessible to the linker. +RUN ln -s /usr/lib/gcc/x86_64-linux-gnu/12/libgccjit.so /usr/lib/x86_64-linux-gnu/libgccjit.so + # We are disabling CI LLVM since this builder is intentionally using a host # LLVM, rather than the typical src/llvm-project LLVM. ENV NO_DOWNLOAD_CI_LLVM 1 @@ -47,6 +51,7 @@ ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ --llvm-root=/usr/lib/llvm-15 \ --enable-llvm-link-shared \ + $USE_NEW_MANGLING \ --set rust.thin-lto-import-instr-limit=10 COPY host-x86_64/x86_64-gnu-llvm-15/script.sh /tmp/ diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh index 918b19612..2eb751ca3 100755 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh @@ -4,34 +4,46 @@ set -ex # Only run the stage 1 tests on merges, not on PR CI jobs. if [[ -z "${PR_CI_JOB}" ]]; then - ../x.py --stage 1 test --skip src/tools/tidy && \ - # Run the `mir-opt` tests again but this time for a 32-bit target. - # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have - # both 32-bit and 64-bit outputs updated by the PR author, before - # the PR is approved and tested for merging. - # It will also detect tests lacking `// EMIT_MIR_FOR_EACH_BIT_WIDTH`, - # despite having different output on 32-bit vs 64-bit targets. - ../x.py --stage 1 test tests/mir-opt \ - --host='' --target=i686-unknown-linux-gnu && \ - # Run `ui-fulldeps` in `--stage=1`, which actually uses the stage0 - # compiler, and is sensitive to the addition of new flags. - ../x.py --stage 1 test tests/ui-fulldeps + # When running gcc backend tests, we need to install `libgccjit` and to not run llvm codegen + # tests as it will fail them. + if [[ "${ENABLE_GCC_CODEGEN}" == "1" ]]; then + ../x.py --stage 1 test --skip src/tools/tidy --skip tests/codegen + else + ../x.py --stage 1 test --skip src/tools/tidy + fi + + # Run the `mir-opt` tests again but this time for a 32-bit target. + # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have + # both 32-bit and 64-bit outputs updated by the PR author, before + # the PR is approved and tested for merging. + # It will also detect tests lacking `// EMIT_MIR_FOR_EACH_BIT_WIDTH`, + # despite having different output on 32-bit vs 64-bit targets. + ../x.py --stage 1 test tests/mir-opt --host='' --target=i686-unknown-linux-gnu + + # Run `ui-fulldeps` in `--stage=1`, which actually uses the stage0 + # compiler, and is sensitive to the addition of new flags. + ../x.py --stage 1 test tests/ui-fulldeps fi +# When running gcc backend tests, we need to install `libgccjit` and to not run llvm codegen +# tests as it will fail them. # NOTE: intentionally uses all of `x.py`, `x`, and `x.ps1` to make sure they all work on Linux. -../x.py --stage 2 test --skip src/tools/tidy && \ - # Run the `mir-opt` tests again but this time for a 32-bit target. - # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have - # both 32-bit and 64-bit outputs updated by the PR author, before - # the PR is approved and tested for merging. - # It will also detect tests lacking `// EMIT_MIR_FOR_EACH_BIT_WIDTH`, - # despite having different output on 32-bit vs 64-bit targets. - ../x --stage 2 test tests/mir-opt \ - --host='' --target=i686-unknown-linux-gnu && \ - # Run the UI test suite again, but in `--pass=check` mode - # - # This is intended to make sure that both `--pass=check` continues to - # work. - # - ../x.ps1 --stage 2 test tests/ui --pass=check \ - --host='' --target=i686-unknown-linux-gnu +if [[ "${ENABLE_GCC_CODEGEN}" == "1" ]]; then + ../x.py --stage 2 test --skip src/tools/tidy --skip tests/codegen +else + ../x.py --stage 2 test --skip src/tools/tidy +fi + +# Run the `mir-opt` tests again but this time for a 32-bit target. +# This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have +# both 32-bit and 64-bit outputs updated by the PR author, before +# the PR is approved and tested for merging. +# It will also detect tests lacking `// EMIT_MIR_FOR_EACH_BIT_WIDTH`, +# despite having different output on 32-bit vs 64-bit targets. +../x --stage 2 test tests/mir-opt --host='' --target=i686-unknown-linux-gnu + +# Run the UI test suite again, but in `--pass=check` mode +# +# This is intended to make sure that both `--pass=check` continues to +# work. +../x.ps1 --stage 2 test tests/ui --pass=check --host='' --target=i686-unknown-linux-gnu diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-16/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-16/Dockerfile index 1e2b802e6..c177e7387 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-16/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-16/Dockerfile @@ -38,6 +38,10 @@ RUN sh /scripts/sccache.sh # LLVM, rather than the typical src/llvm-project LLVM. ENV NO_DOWNLOAD_CI_LLVM 1 +# This is not the latest LLVM version, so some components required by tests may +# be missing. +ENV IS_NOT_LATEST_LLVM 1 + # Using llvm-link-shared due to libffi issues -- see #34486 ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-17/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-17/Dockerfile new file mode 100644 index 000000000..76846f1fe --- /dev/null +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-17/Dockerfile @@ -0,0 +1,50 @@ +FROM ubuntu:23.10 + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + g++ \ + gcc-multilib \ + make \ + ninja-build \ + file \ + curl \ + ca-certificates \ + python3 \ + git \ + cmake \ + sudo \ + gdb \ + llvm-17-tools \ + llvm-17-dev \ + libedit-dev \ + libssl-dev \ + pkg-config \ + zlib1g-dev \ + xz-utils \ + nodejs \ + mingw-w64 \ + && rm -rf /var/lib/apt/lists/* + +# Install powershell (universal package) so we can test x.ps1 on Linux +RUN curl -sL "https://github.com/PowerShell/PowerShell/releases/download/v7.3.1/powershell_7.3.1-1.deb_amd64.deb" > powershell.deb && \ + dpkg -i powershell.deb && \ + rm -f powershell.deb + +COPY scripts/sccache.sh /scripts/ +RUN sh /scripts/sccache.sh + +# We are disabling CI LLVM since this builder is intentionally using a host +# LLVM, rather than the typical src/llvm-project LLVM. +ENV NO_DOWNLOAD_CI_LLVM 1 + +# Using llvm-link-shared due to libffi issues -- see #34486 +ENV RUST_CONFIGURE_ARGS \ + --build=x86_64-unknown-linux-gnu \ + --llvm-root=/usr/lib/llvm-17 \ + --enable-llvm-link-shared \ + --set rust.thin-lto-import-instr-limit=10 + +COPY host-x86_64/x86_64-gnu-llvm-15/script.sh /tmp/ + +ENV SCRIPT /tmp/script.sh diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile index 85f2f84a4..82385ea15 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile @@ -15,6 +15,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ sudo \ xz-utils \ tidy \ + libgccjit-12-dev \ \ # Install dependencies for chromium browser gconf-service \ @@ -61,6 +62,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh +# Make `libgccjit.so` accessible. +RUN ln -s /usr/lib/gcc/x86_64-linux-gnu/12/libgccjit.so /usr/lib/x86_64-linux-gnu/libgccjit.so +# Fix rustc_codegen_gcc lto issues. +ENV GCC_EXEC_PREFIX="/usr/lib/gcc/" + COPY host-x86_64/x86_64-gnu-tools/checktools.sh /tmp/ RUN curl -sL https://nodejs.org/dist/v14.20.0/node-v14.20.0-linux-x64.tar.xz | tar -xJ @@ -81,7 +87,10 @@ RUN npm install -g browser-ui-test@$(head -n 1 /tmp/browser-ui-test.version) --u ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ - --save-toolstates=/tmp/toolstate/toolstates.json + --save-toolstates=/tmp/toolstate/toolstates.json \ + --enable-new-symbol-mangling + +ENV HOST_TARGET x86_64-unknown-linux-gnu ENV SCRIPT /tmp/checktools.sh ../x.py && \ NODE_PATH=`npm root -g` python3 ../x.py test tests/rustdoc-gui --stage 2 \ diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh b/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh index 7dde63709..821a09feb 100755 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh @@ -1,4 +1,5 @@ #!/bin/sh +# ignore-tidy-linelength set -eu @@ -26,8 +27,30 @@ python3 "$X_PY" test --stage 2 src/tools/clippy python3 "$X_PY" test --stage 2 src/tools/rustfmt python3 "$X_PY" test --stage 2 src/tools/miri # We natively run this script on x86_64-unknown-linux-gnu and x86_64-pc-windows-msvc. -# Also cover some other targets (on both of these hosts) via cross-testing. +# Also cover some other targets via cross-testing, in particular all tier 1 targets. export BOOTSTRAP_SKIP_TARGET_SANITY=1 # we don't need `cc` for these targets -python3 "$X_PY" test --stage 2 src/tools/miri --target i686-pc-windows-msvc -python3 "$X_PY" test --stage 2 src/tools/miri --target aarch64-apple-darwin +case $HOST_TARGET in + x86_64-unknown-linux-gnu) + # Only this branch runs in PR CI. + # Fully test all main OSes, including a 32bit target. + python3 "$X_PY" test --stage 2 src/tools/miri --target x86_64-apple-darwin + python3 "$X_PY" test --stage 2 src/tools/miri --target i686-pc-windows-msvc + # Only run "pass" tests for the remaining targets, which is quite a bit faster. + python3 "$X_PY" test --stage 2 src/tools/miri --target x86_64-pc-windows-gnu --test-args pass + python3 "$X_PY" test --stage 2 src/tools/miri --target i686-unknown-linux-gnu --test-args pass + python3 "$X_PY" test --stage 2 src/tools/miri --target aarch64-unknown-linux-gnu --test-args pass + python3 "$X_PY" test --stage 2 src/tools/miri --target s390x-unknown-linux-gnu --test-args pass + ;; + x86_64-pc-windows-msvc) + # Strangely, Linux targets do not work here. cargo always says + # "error: cannot produce cdylib for ... as the target ... does not support these crate types". + # Only run "pass" tests, which is quite a bit faster. + python3 "$X_PY" test --stage 2 src/tools/miri --target aarch64-apple-darwin --test-args pass + python3 "$X_PY" test --stage 2 src/tools/miri --target i686-pc-windows-gnu --test-args pass + ;; + *) + echo "FATAL: unexpected host $HOST_TARGET" + exit 1 + ;; +esac unset BOOTSTRAP_SKIP_TARGET_SANITY diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index e9c155b13..cedbc0390 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -235,7 +235,7 @@ else args="$args --volume /tmp/toolstate:/tmp/toolstate" id=$(id -u) - if [[ "$id" != 0 && "$(docker -v)" =~ ^podman ]]; then + if [[ "$id" != 0 && "$(docker version)" =~ Podman ]]; then # Rootless podman creates a separate user namespace, where an inner # LOCAL_USER_ID will map to a different subuid range on the host. # The "keep-id" mode maps the current UID directly into the container. @@ -264,10 +264,27 @@ else BASE_COMMIT="" fi +SUMMARY_FILE=github-summary.md +touch $objdir/${SUMMARY_FILE} + +extra_env="" +if [ "$ENABLE_GCC_CODEGEN" = "1" ]; then + extra_env="$EXTRA_ENV --env ENABLE_GCC_CODEGEN=1" + # If `ENABLE_GCC_CODEGEN` is set and not empty, we add the `--enable-new-symbol-mangling` + # argument to `RUST_CONFIGURE_ARGS` and set the `GCC_EXEC_PREFIX` environment variable. + # `cg_gcc` doesn't support the legacy mangling so we need to enforce the new one + # if we run `cg_gcc` tests. + extra_env="$EXTRA_ENV --env USE_NEW_MANGLING=--enable-new-symbol-mangling" + # Fix rustc_codegen_gcc lto issues. + extra_env="$EXTRA_ENV --env GCC_EXEC_PREFIX=/usr/lib/gcc/" + echo "Setting extra environment values for docker: $extra_env" +fi + docker \ run \ --workdir /checkout/obj \ --env SRC=/checkout \ + $extra_env \ $args \ --env CARGO_HOME=/cargo \ --env DEPLOY \ @@ -275,6 +292,7 @@ docker \ --env CI \ --env GITHUB_ACTIONS \ --env GITHUB_REF \ + --env GITHUB_STEP_SUMMARY="/checkout/obj/${SUMMARY_FILE}" \ --env TOOLSTATE_REPO_ACCESS_TOKEN \ --env TOOLSTATE_REPO \ --env TOOLSTATE_PUBLISH \ @@ -284,11 +302,14 @@ docker \ --env DIST_TRY_BUILD \ --env PR_CI_JOB \ --env OBJDIR_ON_HOST="$objdir" \ + --env CODEGEN_BACKENDS \ --init \ --rm \ rust-ci \ $command +cat $objdir/${SUMMARY_FILE} >> "${GITHUB_STEP_SUMMARY}" + if [ -f /.dockerenv ]; then rm -rf $objdir docker cp checkout:/checkout/obj $objdir diff --git a/src/ci/docker/scripts/fuchsia-test-runner.py b/src/ci/docker/scripts/fuchsia-test-runner.py index f78d446c8..437b51641 100755 --- a/src/ci/docker/scripts/fuchsia-test-runner.py +++ b/src/ci/docker/scripts/fuchsia-test-runner.py @@ -12,6 +12,7 @@ from dataclasses import dataclass import fcntl import glob import hashlib +import io import json import os import platform @@ -276,27 +277,60 @@ class TestEnvironment: stderr=self.subprocess_output(), ) - # Start emulator - self.log_info("Starting emulator...") - product_bundle = "terminal.qemu-" + self.triple_to_arch(self.target) + # Look up the product bundle transfer manifest. + self.log_info("Looking up the product bundle transfer manifest...") + product_name = "minimal." + self.triple_to_arch(self.target) + fuchsia_version = "14.20230811.2.1" + + # FIXME: We should be able to replace this with the machine parsable + # `ffx --machine json product lookup ...` once F15 is released. + out = subprocess.check_output( + [ + ffx_path, + "product", + "lookup", + product_name, + fuchsia_version, + "--base-url", + "gs://fuchsia/development/" + fuchsia_version, + ], + env=ffx_env, + stderr=self.subprocess_output(), + ) + + self.log_debug(out) + + for line in io.BytesIO(out): + if line.startswith(b"gs://"): + transfer_manifest_url = line.rstrip() + break + else: + raise Exception("Unable to parse transfer manifest") + + # Download the product bundle. + product_bundle_dir = os.path.join(self.tmp_dir(), 'product-bundle') subprocess.check_call( [ ffx_path, - "product-bundle", - "get", - product_bundle, + "product", + "download", + transfer_manifest_url, + product_bundle_dir, + "--force", ], env=ffx_env, stdout=self.subprocess_output(), stderr=self.subprocess_output(), ) + + # Start emulator # FIXME: condition --accel hyper on target arch matching host arch subprocess.check_call( [ ffx_path, "emu", "start", - product_bundle, + product_bundle_dir, "--headless", "--log", self.emulator_log_path(), diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 858ebf72a..da29ffb8e 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -91,6 +91,10 @@ x--expand-yaml-anchors--remove: os: macos-13 # We use the standard runner for now <<: *base-job + - &job-macos-m1 + os: macos-13-xlarge + <<: *base-job + - &job-windows-8c os: windows-2019-8core-32gb <<: *base-job @@ -153,6 +157,10 @@ x--expand-yaml-anchors--remove: run: src/ci/scripts/dump-environment.sh <<: *step + - name: install awscli + run: src/ci/scripts/install-awscli.sh + <<: *step + - name: install sccache run: src/ci/scripts/install-sccache.sh <<: *step @@ -165,6 +173,10 @@ x--expand-yaml-anchors--remove: run: src/ci/scripts/install-clang.sh <<: *step + - name: install tidy + run: src/ci/scripts/install-tidy.sh + <<: *step + - name: install WIX run: src/ci/scripts/install-wix.sh <<: *step @@ -281,6 +293,7 @@ on: - auto - try - try-perf + - automation/bors/try - master pull_request: branches: @@ -322,6 +335,8 @@ jobs: <<: *job-linux-4c - name: x86_64-gnu-llvm-15 + env: + ENABLE_GCC_CODEGEN: "1" <<: *job-linux-16c - name: x86_64-gnu-tools @@ -350,6 +365,8 @@ jobs: <<: *job-linux-8c - name: dist-aarch64-linux + env: + CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-8c - name: dist-android @@ -402,14 +419,19 @@ jobs: - &dist-x86_64-linux name: dist-x86_64-linux + env: + CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-16c - name: dist-x86_64-linux-alt env: IMAGE: dist-x86_64-linux + CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-16c - name: dist-x86_64-musl + env: + CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-8c - name: dist-x86_64-netbsd @@ -427,20 +449,6 @@ jobs: - name: test-various <<: *job-linux-8c - - name: wasm32 - env: - # Running emscripten tests currently requires that we are - # building a nightly toolchain. Otherwise, we cannot pass - # -Zunstable-options to libtest. Normally we workaround this by - # setting RUSTC_BOOTSTRAP in the environment, but that doesn't - # work for emscripten as environment variables are not threaded - # into the compiled code. - # - # For more details see: - # https://emscripten.org/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.html#environment-variables - RUST_CI_OVERRIDE_RELEASE_CHANNEL: nightly - <<: *job-linux-8c - - name: x86_64-gnu <<: *job-linux-4c @@ -468,6 +476,11 @@ jobs: - name: x86_64-gnu-distcheck <<: *job-linux-8c + - name: x86_64-gnu-llvm-17 + env: + RUST_BACKTRACE: 1 + <<: *job-linux-8c + - name: x86_64-gnu-llvm-16 env: RUST_BACKTRACE: 1 @@ -501,6 +514,7 @@ jobs: NO_DEBUG_ASSERTIONS: 1 NO_OVERFLOW_CHECKS: 1 DIST_REQUIRE_ALL_TOOLS: 1 + CODEGEN_BACKENDS: llvm,cranelift <<: *job-macos-xl - name: dist-apple-various @@ -536,17 +550,14 @@ jobs: # This target only needs to support 11.0 and up as nothing else supports the hardware - name: dist-aarch64-apple env: - SCRIPT: ./x.py dist bootstrap --include-default-paths --stage 2 + SCRIPT: ./x.py dist bootstrap --include-default-paths --host=aarch64-apple-darwin --target=aarch64-apple-darwin RUST_CONFIGURE_ARGS: >- - --build=x86_64-apple-darwin - --host=aarch64-apple-darwin - --target=aarch64-apple-darwin --enable-full-tools --enable-sanitizers --enable-profiler - --disable-docs --set rust.jemalloc --set llvm.ninja=false + --set rust.lto=thin RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 SELECT_XCODE: /Applications/Xcode_13.4.1.app USE_XCODE_CLANG: 1 @@ -556,15 +567,26 @@ jobs: NO_DEBUG_ASSERTIONS: 1 NO_OVERFLOW_CHECKS: 1 DIST_REQUIRE_ALL_TOOLS: 1 - # Corresponds to 16K page size - # - # Shouldn't be needed if jemalloc-sys is updated to - # handle this platform like iOS or if we build on - # aarch64-apple-darwin itself. - # - # https://github.com/gnzlbg/jemallocator/blob/c27a859e98e3cb790dc269773d9da71a1e918458/jemalloc-sys/build.rs#L237 - JEMALLOC_SYS_WITH_LG_PAGE: 14 - <<: *job-macos-xl + <<: *job-macos-m1 + + # This target only needs to support 11.0 and up as nothing else supports the hardware + - name: aarch64-apple + env: + SCRIPT: ./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin + RUST_CONFIGURE_ARGS: >- + --enable-sanitizers + --enable-profiler + --set rust.jemalloc + --set llvm.ninja=false + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + SELECT_XCODE: /Applications/Xcode_13.4.1.app + USE_XCODE_CLANG: 1 + MACOSX_DEPLOYMENT_TARGET: 11.0 + MACOSX_STD_DEPLOYMENT_TARGET: 11.0 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + NO_OVERFLOW_CHECKS: 1 + <<: *job-macos-m1 ###################### # Windows Builders # @@ -585,6 +607,7 @@ jobs: - name: x86_64-msvc-ext env: SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo && src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows + HOST_TARGET: x86_64-pc-windows-msvc RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld --save-toolstates=/tmp/toolstate/toolstates.json DEPLOY_TOOLSTATES_JSON: toolstates-windows.json <<: *job-windows-8c @@ -702,12 +725,14 @@ jobs: env: DIST_TRY_BUILD: 1 <<: [*shared-ci-variables, *prod-variables] - if: github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust' + if: github.event_name == 'push' && (((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust') || ((github.ref == 'refs/heads/automation/bors/try') && github.repository == 'rust-lang/rust')) strategy: matrix: include: - &dist-x86_64-linux name: dist-x86_64-linux + env: + CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-16c master: diff --git a/src/ci/run.sh b/src/ci/run.sh index 98f2cdac5..ce0dd6018 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -47,7 +47,8 @@ source "$ci_dir/shared.sh" export CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse -if ! isCI || isCiBranch auto || isCiBranch beta || isCiBranch try || isCiBranch try-perf; then +if ! isCI || isCiBranch auto || isCiBranch beta || isCiBranch try || isCiBranch try-perf || \ + isCiBranch automation/bors/try; then RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set build.print-step-timings --enable-verbose-tests" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set build.metrics" HAS_METRICS=1 @@ -97,12 +98,14 @@ if [ "$DEPLOY$DEPLOY_ALT" = "1" ]; then if [ "$NO_LLVM_ASSERTIONS" = "1" ]; then RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-llvm-assertions" elif [ "$DEPLOY_ALT" != "" ]; then - if [ "$NO_PARALLEL_COMPILER" = "" ]; then - RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.parallel-compiler" + if [ "$ALT_PARALLEL_COMPILER" = "" ]; then + RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.parallel-compiler=false" fi RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-llvm-assertions" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir" fi + + RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=${CODEGEN_BACKENDS:-llvm}" else # We almost always want debug assertions enabled, but sometimes this takes too # long for too little benefit, so we just turn them off. @@ -123,8 +126,15 @@ else RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir" - # Test the Cranelift backend in on CI, but don't ship it. - RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift" + # When running gcc backend tests, we need to install `libgccjit` and to not run llvm codegen + # tests as it will fail them. + if [[ "${ENABLE_GCC_CODEGEN}" == "1" ]]; then + # Test the Cranelift and GCC backends in CI. Bootstrap knows which targets to run tests on. + RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift,gcc" + else + # Test the Cranelift backend in CI. Bootstrap knows which targets to run tests on. + RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift" + fi # We enable this for non-dist builders, since those aren't trying to produce # fresh binaries. We currently don't entirely support distributing a fresh diff --git a/src/ci/scripts/install-awscli.sh b/src/ci/scripts/install-awscli.sh new file mode 100755 index 000000000..b4a239fd3 --- /dev/null +++ b/src/ci/scripts/install-awscli.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# This script downloads and installs the awscli binaries directly from +# Amazon. + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +AWS_VERSION="2.13.25" + +# Only the macOS arm64/aarch64 GitHub Actions runner needs to have AWS +# installed; other platforms have it preinstalled. + +if isMacOS; then + platform=$(uname -m) + case $platform in + x86_64) + ;; + arm64) + file="https://awscli.amazonaws.com/AWSCLIV2-${AWS_VERSION}.pkg" + retry curl -f "${file}" -o "AWSCLIV2.pkg" + sudo installer -pkg "AWSCLIV2.pkg" -target / + ;; + *) + echo "unsupported architecture: ${platform}" + exit 1 + esac +fi diff --git a/src/ci/scripts/install-tidy.sh b/src/ci/scripts/install-tidy.sh new file mode 100755 index 000000000..fab126453 --- /dev/null +++ b/src/ci/scripts/install-tidy.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# This script downloads and installs the tidy binary from Homebrew. + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +# Only the macOS arm64/aarch64 GitHub Actions runner needs to have tidy +# installed; other platforms have it preinstalled. + +if isMacOS; then + platform=$(uname -m) + case $platform in + x86_64) + ;; + arm64) + brew install tidy-html5 + ;; + *) + echo "unsupported architecture: ${platform}" + exit 1 + esac +fi diff --git a/src/ci/scripts/verify-channel.sh b/src/ci/scripts/verify-channel.sh index cd28748a4..edeea2014 100755 --- a/src/ci/scripts/verify-channel.sh +++ b/src/ci/scripts/verify-channel.sh @@ -8,7 +8,7 @@ IFS=$'\n\t' source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" -if isCiBranch auto || isCiBranch try || isCiBranch try-perf; then +if isCiBranch auto || isCiBranch try || isCiBranch try-perf || isCiBranch automation/bors/try; then echo "channel verification is only executed on PR builds" exit fi |