From 1376c5a617be5c25655d0d7cb63e3beaa5a6e026 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 17 Apr 2024 14:20:39 +0200 Subject: Merging upstream version 1.70.0+dfsg1. Signed-off-by: Daniel Baumann --- .../host-x86_64/dist-x86_64-illumos/Dockerfile | 9 +- .../host-x86_64/dist-x86_64-linux/Dockerfile | 4 +- .../host-x86_64/dist-x86_64-linux/build-clang.sh | 2 +- .../host-x86_64/dist-x86_64-linux/build-gcc.sh | 2 +- src/ci/docker/host-x86_64/i686-gnu/Dockerfile | 1 + .../docker/host-x86_64/mingw-check-tidy/Dockerfile | 5 +- src/ci/docker/host-x86_64/mingw-check/Dockerfile | 4 +- .../host-x86_64/x86_64-gnu-distcheck/Dockerfile | 2 +- .../x86_64-gnu-llvm-14-stage1/Dockerfile | 1 + .../host-x86_64/x86_64-gnu-llvm-14/Dockerfile | 9 +- .../host-x86_64/x86_64-gnu-llvm-15/Dockerfile | 1 + .../x86_64-gnu-tools/browser-ui-test.version | 2 +- src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile | 1 + src/ci/docker/scripts/emscripten.sh | 4 +- src/ci/github-actions/ci.yml | 144 +++++++++++---------- src/ci/github-actions/problem_matchers.json | 15 +++ src/ci/run.sh | 6 + src/ci/scripts/collect-cpu-stats.sh | 3 +- src/ci/scripts/install-awscli.sh | 7 +- src/ci/scripts/run-build-from-ci.sh | 2 + src/ci/scripts/upload-artifacts.sh | 2 +- src/ci/stage-build.py | 10 +- 22 files changed, 139 insertions(+), 97 deletions(-) create mode 100644 src/ci/github-actions/problem_matchers.json (limited to 'src/ci') diff --git a/src/ci/docker/host-x86_64/dist-x86_64-illumos/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-illumos/Dockerfile index 4e46bdee5..2089bf387 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-illumos/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-illumos/Dockerfile @@ -9,10 +9,10 @@ RUN bash /tmp/cross-apt-packages.sh # Required for cross-build gcc RUN apt-get update && \ apt-get install -y --no-install-recommends \ - libgmp-dev \ - libmpfr-dev \ - libmpc-dev \ - && rm -rf /var/lib/apt/lists/* + libgmp-dev \ + libmpfr-dev \ + libmpc-dev \ + && rm -rf /var/lib/apt/lists/* COPY scripts/illumos-toolchain.sh /tmp/ @@ -28,6 +28,7 @@ RUN /scripts/cmake.sh ENV \ AR_x86_64_unknown_illumos=x86_64-illumos-ar \ + RANLIB_x86_64_unknown_illumos=x86_64-illumos-ranlib \ CC_x86_64_unknown_illumos=x86_64-illumos-gcc \ CXX_x86_64_unknown_illumos=x86_64-illumos-g++ diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index 5feba4e06..04fdb15f5 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -54,8 +54,8 @@ COPY host-x86_64/dist-x86_64-linux/build-clang.sh /tmp/ RUN ./build-clang.sh ENV CC=clang CXX=clang++ -# rustc-perf version from 2022-07-22 -ENV PERF_COMMIT 3c253134664fdcba862c539d37f0de18557a9a4c +# rustc-perf version from 2023-03-15 +ENV PERF_COMMIT 9dfaa35193154b690922347ee1141a06ec87a199 RUN curl -LS -o perf.zip https://github.com/rust-lang/rustc-perf/archive/$PERF_COMMIT.zip && \ unzip perf.zip && \ mv rustc-perf-$PERF_COMMIT rustc-perf && \ diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh b/src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh index 9abfd4e97..9b274cc27 100755 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh @@ -4,7 +4,7 @@ set -ex source shared.sh -LLVM=llvmorg-15.0.0 +LLVM=llvmorg-16.0.0 mkdir llvm-project cd llvm-project diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh b/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh index 9932b2505..41ca1385c 100755 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh @@ -3,7 +3,7 @@ set -ex source shared.sh -GCC=7.5.0 +GCC=8.5.0 curl https://ftp.gnu.org/gnu/gcc/gcc-$GCC/gcc-$GCC.tar.xz | xzcat | tar xf - cd gcc-$GCC diff --git a/src/ci/docker/host-x86_64/i686-gnu/Dockerfile b/src/ci/docker/host-x86_64/i686-gnu/Dockerfile index d9e583862..b5abf6564 100644 --- a/src/ci/docker/host-x86_64/i686-gnu/Dockerfile +++ b/src/ci/docker/host-x86_64/i686-gnu/Dockerfile @@ -16,6 +16,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ zlib1g-dev \ lib32z1-dev \ xz-utils \ + mingw-w64 \ && rm -rf /var/lib/apt/lists/* diff --git a/src/ci/docker/host-x86_64/mingw-check-tidy/Dockerfile b/src/ci/docker/host-x86_64/mingw-check-tidy/Dockerfile index 889a586b3..34b93be41 100644 --- a/src/ci/docker/host-x86_64/mingw-check-tidy/Dockerfile +++ b/src/ci/docker/host-x86_64/mingw-check-tidy/Dockerfile @@ -8,6 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ file \ curl \ ca-certificates \ + python2.7 \ python3 \ python3-pip \ python3-pkg-resources \ @@ -30,4 +31,6 @@ RUN pip3 install --no-deps --no-cache-dir --require-hashes -r /tmp/reuse-require COPY host-x86_64/mingw-check/validate-toolstate.sh /scripts/ COPY host-x86_64/mingw-check/validate-error-codes.sh /scripts/ -ENV SCRIPT python3 ../x.py test --stage 0 src/tools/tidy tidyselftest +# NOTE: intentionally uses python2 for x.py so we can test it still works. +# validate-toolstate only runs in our CI, so it's ok for it to only support python3. +ENV SCRIPT python2.7 ../x.py test --stage 0 src/tools/tidy tidyselftest diff --git a/src/ci/docker/host-x86_64/mingw-check/Dockerfile b/src/ci/docker/host-x86_64/mingw-check/Dockerfile index 98bd90210..515890aef 100644 --- a/src/ci/docker/host-x86_64/mingw-check/Dockerfile +++ b/src/ci/docker/host-x86_64/mingw-check/Dockerfile @@ -52,4 +52,6 @@ ENV SCRIPT python3 ../x.py --stage 2 test src/tools/expand-yaml-anchors && \ reuse lint && \ # Runs checks to ensure that there are no ES5 issues in our JS code. es-check es6 ../src/librustdoc/html/static/js/*.js && \ - eslint -c ../src/librustdoc/html/static/.eslintrc.js ../src/librustdoc/html/static/js/*.js + eslint -c ../src/librustdoc/html/static/.eslintrc.js ../src/librustdoc/html/static/js/*.js && \ + eslint -c ../src/tools/rustdoc-js/.eslintrc.js ../src/tools/rustdoc-js/tester.js && \ + eslint -c ../src/tools/rustdoc-gui/.eslintrc.js ../src/tools/rustdoc-gui/tester.js diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile index 7e640c49f..2217e6ee7 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile @@ -24,6 +24,6 @@ RUN sh /scripts/sccache.sh # We are disabling CI LLVM since distcheck is an offline build. ENV NO_DOWNLOAD_CI_LLVM 1 -ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu --set rust.ignore-git=false +ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu --set rust.omit-git-hash=false ENV SCRIPT python3 ../x.py --stage 2 test distcheck ENV DIST_SRC 1 diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-14-stage1/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-14-stage1/Dockerfile index b99a0886b..21dcf29b4 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-14-stage1/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-14-stage1/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ zlib1g-dev \ xz-utils \ nodejs \ + mingw-w64 \ && rm -rf /var/lib/apt/lists/* COPY scripts/sccache.sh /scripts/ diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-14/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-14/Dockerfile index db6032f87..cfb638e8b 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-14/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-14/Dockerfile @@ -2,7 +2,6 @@ FROM ubuntu:22.04 ARG DEBIAN_FRONTEND=noninteractive -# NOTE: intentionally installs both python2 and python3 so we can test support for both. RUN apt-get update && apt-get install -y --no-install-recommends \ g++ \ gcc-multilib \ @@ -11,8 +10,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ file \ curl \ ca-certificates \ - python2.7 \ - python3 \ + python3.11 \ git \ cmake \ sudo \ @@ -25,6 +23,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ zlib1g-dev \ xz-utils \ nodejs \ + mingw-w64 \ && rm -rf /var/lib/apt/lists/* # Install powershell (universal package) so we can test x.ps1 on Linux @@ -62,6 +61,4 @@ ENV SCRIPT ../x.py --stage 2 test --exclude src/tools/tidy && \ # work. # ../x.ps1 --stage 2 test tests/ui --pass=check \ - --host='' --target=i686-unknown-linux-gnu && \ - # Run tidy at the very end, after all the other tests. - python2.7 ../x.py --stage 2 test src/tools/tidy + --host='' --target=i686-unknown-linux-gnu diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile index 5219247cc..fb5037e3b 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile @@ -25,6 +25,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ zlib1g-dev \ xz-utils \ nodejs \ + mingw-w64 \ && rm -rf /var/lib/apt/lists/* # Install powershell (universal package) so we can test x.ps1 on Linux diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version b/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version index 9c550b2d7..7092c7c46 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version @@ -1 +1 @@ -0.14.4 \ No newline at end of file +0.15.0 \ No newline at end of file diff --git a/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile index 5b9581f72..fbec368c9 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile @@ -16,6 +16,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libssl-dev \ pkg-config \ xz-utils \ + mingw-w64 \ && rm -rf /var/lib/apt/lists/* COPY scripts/sccache.sh /scripts/ diff --git a/src/ci/docker/scripts/emscripten.sh b/src/ci/docker/scripts/emscripten.sh index 56dc96283..3f5e2c6ff 100644 --- a/src/ci/docker/scripts/emscripten.sh +++ b/src/ci/docker/scripts/emscripten.sh @@ -20,5 +20,5 @@ exit 1 git clone https://github.com/emscripten-core/emsdk.git /emsdk-portable cd /emsdk-portable -hide_output ./emsdk install 1.39.20 -./emsdk activate 1.39.20 +hide_output ./emsdk install 2.0.5 +./emsdk activate 2.0.5 diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index c594288dc..bfca7b3ab 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -73,18 +73,26 @@ x--expand-yaml-anchors--remove: - &base-job env: {} - - &job-linux-xl + - &job-linux-8c + os: ubuntu-20.04-8core-32gb + <<: *base-job + + - &job-linux-16c os: ubuntu-20.04-16core-64gb <<: *base-job - &job-macos-xl - os: macos-12-xl + os: macos-latest # We use the standard runner for now <<: *base-job - - &job-windows-xl + - &job-windows-8c os: windows-2019-8core-32gb <<: *base-job + - &job-windows-16c + os: windows-2019-16core-64gb + <<: *base-job + - &job-aarch64-linux os: [self-hosted, ARM64, linux] @@ -284,7 +292,7 @@ jobs: permissions: actions: write # for rust-lang/simpleinfra/github-actions/cancel-outdated-builds <<: *base-ci-job - name: PR + name: PR - ${{ matrix.name }} env: <<: [*shared-ci-variables, *public-variables] if: github.event_name == 'pull_request' @@ -293,26 +301,26 @@ jobs: matrix: include: - name: mingw-check - <<: *job-linux-xl + <<: *job-linux-16c tidy: false - name: mingw-check-tidy - <<: *job-linux-xl + <<: *job-linux-16c tidy: true - name: x86_64-gnu-llvm-14 - <<: *job-linux-xl + <<: *job-linux-16c tidy: false - name: x86_64-gnu-tools - <<: *job-linux-xl + <<: *job-linux-16c tidy: false auto: permissions: actions: write # for rust-lang/simpleinfra/github-actions/cancel-outdated-builds <<: *base-ci-job - name: auto + name: auto - ${{ matrix.name }} env: <<: [*shared-ci-variables, *prod-variables] if: github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust' @@ -327,103 +335,103 @@ jobs: <<: *job-aarch64-linux - name: arm-android - <<: *job-linux-xl + <<: *job-linux-8c - name: armhf-gnu - <<: *job-linux-xl + <<: *job-linux-8c - name: dist-aarch64-linux - <<: *job-linux-xl + <<: *job-linux-8c - name: dist-android - <<: *job-linux-xl + <<: *job-linux-8c - name: dist-arm-linux - <<: *job-linux-xl + <<: *job-linux-16c - name: dist-armhf-linux - <<: *job-linux-xl + <<: *job-linux-8c - name: dist-armv7-linux - <<: *job-linux-xl + <<: *job-linux-8c - name: dist-i586-gnu-i586-i686-musl - <<: *job-linux-xl + <<: *job-linux-8c - name: dist-i686-linux - <<: *job-linux-xl + <<: *job-linux-8c - name: dist-mips-linux - <<: *job-linux-xl + <<: *job-linux-8c - name: dist-mips64-linux - <<: *job-linux-xl + <<: *job-linux-8c - name: dist-mips64el-linux - <<: *job-linux-xl + <<: *job-linux-8c - name: dist-mipsel-linux - <<: *job-linux-xl + <<: *job-linux-8c - name: dist-powerpc-linux - <<: *job-linux-xl + <<: *job-linux-8c - name: dist-powerpc64-linux - <<: *job-linux-xl + <<: *job-linux-8c - name: dist-powerpc64le-linux - <<: *job-linux-xl + <<: *job-linux-8c - name: dist-riscv64-linux - <<: *job-linux-xl + <<: *job-linux-8c - name: dist-s390x-linux - <<: *job-linux-xl + <<: *job-linux-8c - name: dist-various-1 - <<: *job-linux-xl + <<: *job-linux-8c - name: dist-various-2 - <<: *job-linux-xl + <<: *job-linux-8c - name: dist-x86_64-freebsd - <<: *job-linux-xl + <<: *job-linux-8c - name: dist-x86_64-illumos - <<: *job-linux-xl + <<: *job-linux-8c - &dist-x86_64-linux name: dist-x86_64-linux - <<: *job-linux-xl + <<: *job-linux-16c - name: dist-x86_64-linux-alt env: IMAGE: dist-x86_64-linux - <<: *job-linux-xl + <<: *job-linux-16c - name: dist-x86_64-musl - <<: *job-linux-xl + <<: *job-linux-8c - name: dist-x86_64-netbsd - <<: *job-linux-xl + <<: *job-linux-8c - name: i686-gnu - <<: *job-linux-xl + <<: *job-linux-16c - name: i686-gnu-nopt - <<: *job-linux-xl + <<: *job-linux-16c - name: mingw-check - <<: *job-linux-xl + <<: *job-linux-8c - name: test-various - <<: *job-linux-xl + <<: *job-linux-8c - name: wasm32 - <<: *job-linux-xl + <<: *job-linux-8c - name: x86_64-gnu - <<: *job-linux-xl + <<: *job-linux-8c # This job ensures commits landing on nightly still pass the full # test suite on the stable channel. There are some UI tests that @@ -438,39 +446,39 @@ jobs: # could cause failures when `dev: 1` in `stage0.txt`, and running # this on stable is useless. CI_ONLY_WHEN_CHANNEL: nightly - <<: *job-linux-xl + <<: *job-linux-8c - name: x86_64-gnu-aux - <<: *job-linux-xl + <<: *job-linux-8c - name: x86_64-gnu-debug - <<: *job-linux-xl + <<: *job-linux-8c - name: x86_64-gnu-distcheck - <<: *job-linux-xl + <<: *job-linux-8c - name: x86_64-gnu-llvm-15 env: RUST_BACKTRACE: 1 - <<: *job-linux-xl + <<: *job-linux-8c - name: x86_64-gnu-llvm-14 env: RUST_BACKTRACE: 1 - <<: *job-linux-xl + <<: *job-linux-8c - name: x86_64-gnu-llvm-14-stage1 env: RUST_BACKTRACE: 1 - <<: *job-linux-xl + <<: *job-linux-8c - name: x86_64-gnu-nopt - <<: *job-linux-xl + <<: *job-linux-8c - name: x86_64-gnu-tools env: DEPLOY_TOOLSTATES_JSON: toolstates-linux.json - <<: *job-linux-xl + <<: *job-linux-8c #################### # macOS Builders # @@ -572,38 +580,38 @@ jobs: env: RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler SCRIPT: make ci-subset-1 - <<: *job-windows-xl + <<: *job-windows-8c - name: x86_64-msvc-2 env: RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler SCRIPT: make ci-subset-2 - <<: *job-windows-xl + <<: *job-windows-8c - name: i686-msvc-1 env: RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc SCRIPT: make ci-subset-1 - <<: *job-windows-xl + <<: *job-windows-8c - name: i686-msvc-2 env: RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc SCRIPT: make ci-subset-2 - <<: *job-windows-xl + <<: *job-windows-8c - name: x86_64-msvc-cargo env: SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld - <<: *job-windows-xl + <<: *job-windows-8c - name: x86_64-msvc-tools env: SCRIPT: src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json DEPLOY_TOOLSTATES_JSON: toolstates-windows.json - <<: *job-windows-xl + <<: *job-windows-8c # 32/64-bit MinGW builds. # @@ -629,7 +637,7 @@ jobs: # incompatible with LLVM downloads today). NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 - <<: *job-windows-xl + <<: *job-windows-8c - name: i686-mingw-2 env: @@ -639,7 +647,7 @@ jobs: # incompatible with LLVM downloads today). NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 - <<: *job-windows-xl + <<: *job-windows-8c - name: x86_64-mingw-1 env: @@ -651,7 +659,7 @@ jobs: # incompatible with LLVM downloads today). NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 - <<: *job-windows-xl + <<: *job-windows-8c - name: x86_64-mingw-2 env: @@ -663,7 +671,7 @@ jobs: # incompatible with LLVM downloads today). NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 - <<: *job-windows-xl + <<: *job-windows-8c - name: dist-x86_64-msvc env: @@ -675,7 +683,7 @@ jobs: --enable-profiler SCRIPT: PGO_HOST=x86_64-pc-windows-msvc python src/ci/stage-build.py python x.py dist bootstrap --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 - <<: *job-windows-xl + <<: *job-windows-8c - name: dist-i686-msvc env: @@ -687,7 +695,7 @@ jobs: --enable-profiler SCRIPT: python x.py dist bootstrap --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 - <<: *job-windows-xl + <<: *job-windows-8c - name: dist-aarch64-msvc env: @@ -701,7 +709,7 @@ jobs: # Hack around this SDK version, because it doesn't work with clang. # See https://github.com/rust-lang/rust/issues/88796 WINDOWS_SDK_20348_HACK: 1 - <<: *job-windows-xl + <<: *job-windows-8c - name: dist-i686-mingw env: @@ -715,7 +723,7 @@ jobs: SCRIPT: python x.py dist bootstrap --include-default-paths CUSTOM_MINGW: 1 DIST_REQUIRE_ALL_TOOLS: 1 - <<: *job-windows-xl + <<: *job-windows-8c - name: dist-x86_64-mingw env: @@ -729,19 +737,19 @@ jobs: NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 DIST_REQUIRE_ALL_TOOLS: 1 - <<: *job-windows-xl + <<: *job-windows-8c - name: dist-x86_64-msvc-alt env: RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler SCRIPT: python x.py dist bootstrap --include-default-paths - <<: *job-windows-xl + <<: *job-windows-8c try: permissions: actions: write # for rust-lang/simpleinfra/github-actions/cancel-outdated-builds <<: *base-ci-job - name: try + name: try - ${{ matrix.name }} env: <<: [*shared-ci-variables, *prod-variables] if: github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust' @@ -750,7 +758,7 @@ jobs: include: - &dist-x86_64-linux name: dist-x86_64-linux - <<: *job-linux-xl + <<: *job-linux-16c master: name: master diff --git a/src/ci/github-actions/problem_matchers.json b/src/ci/github-actions/problem_matchers.json new file mode 100644 index 000000000..37561924b --- /dev/null +++ b/src/ci/github-actions/problem_matchers.json @@ -0,0 +1,15 @@ +{ + "problemMatcher": [ + { + "owner": "tidy-error-file-line", + "pattern": [ + { + "regexp": "^tidy error: /checkout/(.+):(\\d+): (.+)$", + "file": 1, + "line": 2, + "message": 3 + } + ] + } + ] +} diff --git a/src/ci/run.sh b/src/ci/run.sh index efeb850cd..3056d9fc0 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -58,6 +58,12 @@ RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-manage-submodules" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-locked-deps" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-cargo-native-static" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-units-std=1" +# rust-lang/promote-release will recompress CI artifacts, and while we care +# about the per-commit artifact sizes, it's not as important that they're +# highly compressed as it is that the process is fast. Best compression +# generally implies single-threaded compression which results in wasting most +# of our CPU resources. +RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set dist.compression-profile=balanced" # When building for mingw, limit the number of parallel linker jobs during # the LLVM build, as not to run out of memory. diff --git a/src/ci/scripts/collect-cpu-stats.sh b/src/ci/scripts/collect-cpu-stats.sh index 853b4628f..44875b54d 100755 --- a/src/ci/scripts/collect-cpu-stats.sh +++ b/src/ci/scripts/collect-cpu-stats.sh @@ -6,4 +6,5 @@ set -euo pipefail IFS=$'\n\t' -python3 src/ci/cpu-usage-over-time.py &> cpu-usage.csv & +mkdir -p build +python3 src/ci/cpu-usage-over-time.py &> build/cpu-usage.csv & diff --git a/src/ci/scripts/install-awscli.sh b/src/ci/scripts/install-awscli.sh index 3d8f0de7a..aa62407ea 100755 --- a/src/ci/scripts/install-awscli.sh +++ b/src/ci/scripts/install-awscli.sh @@ -10,15 +10,14 @@ # # Before compressing please make sure all the wheels end with `-none-any.whl`. # If that's not the case you'll need to remove the non-cross-platform ones and -# replace them with the .tar.gz downloaded from https://pypi.org. Also make -# sure it's possible to call this script with both Python 2 and Python 3. +# replace them with the .tar.gz downloaded from https://pypi.org. set -euo pipefail IFS=$'\n\t' source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" -MIRROR="${MIRRORS_BASE}/2019-07-27-awscli.tar" +MIRROR="${MIRRORS_BASE}/2023-04-28-awscli.tar" DEPS_DIR="/tmp/awscli-deps" pip="pip" @@ -29,6 +28,8 @@ if isLinux; then sudo apt-get install -y python3-setuptools python3-wheel ciCommandAddPath "${HOME}/.local/bin" +elif isMacOS; then + pip="pip3" fi mkdir -p "${DEPS_DIR}" diff --git a/src/ci/scripts/run-build-from-ci.sh b/src/ci/scripts/run-build-from-ci.sh index c02117f45..55e75800d 100755 --- a/src/ci/scripts/run-build-from-ci.sh +++ b/src/ci/scripts/run-build-from-ci.sh @@ -10,6 +10,8 @@ source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" export CI="true" export SRC=. +echo "::add-matcher::src/ci/github-actions/problem_matchers.json" + # Remove any preexisting rustup installation since it can interfere # with the cargotest step and its auto-detection of things like Clippy in # the environment diff --git a/src/ci/scripts/upload-artifacts.sh b/src/ci/scripts/upload-artifacts.sh index ffa1859fc..9755edb6d 100755 --- a/src/ci/scripts/upload-artifacts.sh +++ b/src/ci/scripts/upload-artifacts.sh @@ -23,7 +23,7 @@ if [[ "${DEPLOY-0}" -eq "1" ]] || [[ "${DEPLOY_ALT-0}" -eq "1" ]]; then fi # CPU usage statistics. -cp cpu-usage.csv "${upload_dir}/cpu-${CI_JOB_NAME}.csv" +cp build/cpu-usage.csv "${upload_dir}/cpu-${CI_JOB_NAME}.csv" # Build metrics generated by x.py. cp "${build_dir}/metrics.json" "${upload_dir}/metrics-${CI_JOB_NAME}.json" diff --git a/src/ci/stage-build.py b/src/ci/stage-build.py index bd8fd524a..7cd5e88f6 100644 --- a/src/ci/stage-build.py +++ b/src/ci/stage-build.py @@ -175,8 +175,8 @@ class WindowsPipeline(Pipeline): return super().rustc_stage_2().with_suffix(".exe") def build_rustc_perf(self): - # rustc-perf version from 2022-07-22 - perf_commit = "3c253134664fdcba862c539d37f0de18557a9a4c" + # rustc-perf version from 2023-03-15 + perf_commit = "9dfaa35193154b690922347ee1141a06ec87a199" rustc_perf_zip_path = self.opt_artifacts() / "perf.zip" def download_rustc_perf(): @@ -727,7 +727,7 @@ def record_metrics(pipeline: Pipeline, timer: Timer): metrics = load_last_metrics(pipeline.metrics_path()) if metrics is None: return - llvm_steps = tuple(metrics.find_all_by_type("bootstrap::native::Llvm")) + llvm_steps = tuple(metrics.find_all_by_type("bootstrap::llvm::Llvm")) assert len(llvm_steps) > 0 llvm_duration = sum(step.duration for step in llvm_steps) @@ -798,14 +798,16 @@ def execute_build_pipeline(timer: Timer, pipeline: Pipeline, final_build_args: L "--llvm-profile-use", pipeline.llvm_profile_merged_file(), "--llvm-bolt-profile-generate", + "--rust-profile-use", + pipeline.rustc_profile_merged_file() ]) record_metrics(pipeline, rustc_build) with stage3.section("Gather profiles"): gather_llvm_bolt_profiles(pipeline) + # LLVM is not being cleared here, we want to reuse the previous build print_free_disk_space(pipeline) - clear_llvm_files(pipeline) final_build_args += [ "--llvm-bolt-profile-use", pipeline.llvm_bolt_profile_merged_file() -- cgit v1.2.3