From 3e3e70d529d8c7d7c4d7bc4fefc9f109393b9245 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 17 Apr 2024 14:19:43 +0200 Subject: Merging upstream version 1.69.0+dfsg1. Signed-off-by: Daniel Baumann --- .../docker/host-x86_64/dist-s390x-linux/Dockerfile | 2 +- .../host-x86_64/dist-x86_64-linux/Dockerfile | 2 +- src/ci/docker/host-x86_64/mingw-check/Dockerfile | 4 +- .../host-x86_64/mingw-check/validate-toolstate.sh | 6 - .../x86_64-gnu-llvm-13-stage1/Dockerfile | 49 -- .../host-x86_64/x86_64-gnu-llvm-13/Dockerfile | 69 -- .../x86_64-gnu-llvm-14-stage1/Dockerfile | 49 ++ .../host-x86_64/x86_64-gnu-llvm-14/Dockerfile | 67 ++ .../host-x86_64/x86_64-gnu-llvm-15/Dockerfile | 67 ++ .../x86_64-gnu-tools/browser-ui-test.version | 2 +- src/ci/docker/scripts/fuchsia-test-runner.py | 172 +++-- src/ci/github-actions/ci.yml | 34 +- src/ci/pgo.sh | 230 ------ src/ci/run.sh | 10 + src/ci/scripts/install-mingw.sh | 23 +- src/ci/scripts/should-skip-this.sh | 37 +- src/ci/stage-build.py | 842 +++++++++++++++++++++ 17 files changed, 1177 insertions(+), 488 deletions(-) delete mode 100644 src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile delete mode 100644 src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile create mode 100644 src/ci/docker/host-x86_64/x86_64-gnu-llvm-14-stage1/Dockerfile create mode 100644 src/ci/docker/host-x86_64/x86_64-gnu-llvm-14/Dockerfile create mode 100644 src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile delete mode 100755 src/ci/pgo.sh create mode 100644 src/ci/stage-build.py (limited to 'src/ci') diff --git a/src/ci/docker/host-x86_64/dist-s390x-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-s390x-linux/Dockerfile index 43a449b3a..adb98d7eb 100644 --- a/src/ci/docker/host-x86_64/dist-s390x-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-s390x-linux/Dockerfile @@ -28,5 +28,5 @@ ENV \ ENV HOSTS=s390x-unknown-linux-gnu -ENV RUST_CONFIGURE_ARGS --enable-extended --enable-lld --enable-profiler --disable-docs +ENV RUST_CONFIGURE_ARGS --enable-extended --enable-lld --enable-sanitizers --enable-profiler --disable-docs ENV SCRIPT python3 ../x.py dist --host $HOSTS --target $HOSTS diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index 6bdc88e18..5feba4e06 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -81,7 +81,7 @@ ENV RUST_CONFIGURE_ARGS \ --set rust.jemalloc \ --set rust.use-lld=true \ --set rust.lto=thin -ENV SCRIPT ../src/ci/pgo.sh python3 ../x.py dist \ +ENV SCRIPT python3 ../src/ci/stage-build.py python3 ../x.py dist \ --host $HOSTS --target $HOSTS \ --include-default-paths \ build-manifest bootstrap diff --git a/src/ci/docker/host-x86_64/mingw-check/Dockerfile b/src/ci/docker/host-x86_64/mingw-check/Dockerfile index 4cc5d9f8a..98bd90210 100644 --- a/src/ci/docker/host-x86_64/mingw-check/Dockerfile +++ b/src/ci/docker/host-x86_64/mingw-check/Dockerfile @@ -23,6 +23,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ RUN curl -sL https://nodejs.org/dist/v16.9.0/node-v16.9.0-linux-x64.tar.xz | tar -xJ ENV PATH="/node-v16.9.0-linux-x64/bin:${PATH}" +ENV RUST_CONFIGURE_ARGS="--set rust.validate-mir-opts=3" + # Install es-check # Pin its version to prevent unrelated CI failures due to future es-check versions. RUN npm install es-check@6.1.1 eslint@8.6.0 -g @@ -38,7 +40,7 @@ COPY host-x86_64/mingw-check/validate-error-codes.sh /scripts/ ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1 ENV SCRIPT python3 ../x.py --stage 2 test src/tools/expand-yaml-anchors && \ - python3 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu --all-targets && \ + python3 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu && \ python3 ../x.py build --stage 0 src/tools/build-manifest && \ python3 ../x.py test --stage 0 src/tools/compiletest && \ python3 ../x.py test --stage 0 core alloc std test proc_macro && \ diff --git a/src/ci/docker/host-x86_64/mingw-check/validate-toolstate.sh b/src/ci/docker/host-x86_64/mingw-check/validate-toolstate.sh index c6d728eb8..0b06f5e36 100755 --- a/src/ci/docker/host-x86_64/mingw-check/validate-toolstate.sh +++ b/src/ci/docker/host-x86_64/mingw-check/validate-toolstate.sh @@ -9,11 +9,5 @@ git clone --depth=1 https://github.com/rust-lang-nursery/rust-toolstate.git cd rust-toolstate python3 "../../src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" \ "$(git log --format=%s -n1 HEAD)" "" "" -# Only check maintainers if this build is supposed to publish toolstate. -# Builds that are not supposed to publish don't have the access token. -if [ -n "${TOOLSTATE_PUBLISH+is_set}" ]; then - TOOLSTATE_VALIDATE_MAINTAINERS_REPO=rust-lang/rust python3 \ - "../../src/tools/publish_toolstate.py" -fi cd .. rm -rf rust-toolstate diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile deleted file mode 100644 index bcbf58253..000000000 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -FROM ubuntu:22.04 - -ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends \ - g++ \ - gcc-multilib \ - make \ - ninja-build \ - file \ - curl \ - ca-certificates \ - python2.7 \ - git \ - cmake \ - sudo \ - gdb \ - llvm-13-tools \ - llvm-13-dev \ - libedit-dev \ - libssl-dev \ - pkg-config \ - zlib1g-dev \ - xz-utils \ - nodejs \ - && rm -rf /var/lib/apt/lists/* - -COPY scripts/sccache.sh /scripts/ -RUN sh /scripts/sccache.sh - -# We are disabling CI LLVM since this builder is intentionally using a host -# LLVM, rather than the typical src/llvm-project LLVM. -ENV NO_DOWNLOAD_CI_LLVM 1 - -# Using llvm-link-shared due to libffi issues -- see #34486 -ENV RUST_CONFIGURE_ARGS \ - --build=x86_64-unknown-linux-gnu \ - --llvm-root=/usr/lib/llvm-13 \ - --enable-llvm-link-shared \ - --set rust.thin-lto-import-instr-limit=10 - -ENV SCRIPT python2.7 ../x.py --stage 1 test --exclude src/tools/tidy && \ - # Run the `mir-opt` tests again but this time for a 32-bit target. - # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have - # both 32-bit and 64-bit outputs updated by the PR author, before - # the PR is approved and tested for merging. - # It will also detect tests lacking `// EMIT_MIR_FOR_EACH_BIT_WIDTH`, - # despite having different output on 32-bit vs 64-bit targets. - python2.7 ../x.py --stage 1 test tests/mir-opt \ - --host='' --target=i686-unknown-linux-gnu diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile deleted file mode 100644 index 9fc9e9cbf..000000000 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile +++ /dev/null @@ -1,69 +0,0 @@ -FROM ubuntu:22.04 - -ARG DEBIAN_FRONTEND=noninteractive - -# NOTE: intentionally installs both python2 and python3 so we can test support for both. -RUN apt-get update && apt-get install -y --no-install-recommends \ - g++ \ - gcc-multilib \ - make \ - ninja-build \ - file \ - curl \ - ca-certificates \ - python2.7 \ - python3.9 \ - git \ - cmake \ - sudo \ - gdb \ - llvm-13-tools \ - llvm-13-dev \ - libedit-dev \ - libssl-dev \ - pkg-config \ - zlib1g-dev \ - xz-utils \ - nodejs \ - \ -# Install powershell so we can test x.ps1 on Linux - apt-transport-https software-properties-common && \ - curl -s "https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/packages-microsoft-prod.deb" > packages-microsoft-prod.deb && \ - dpkg -i packages-microsoft-prod.deb && \ - apt-get update && \ - apt-get install -y powershell \ - && rm -rf /var/lib/apt/lists/* - -COPY scripts/sccache.sh /scripts/ -RUN sh /scripts/sccache.sh - -# We are disabling CI LLVM since this builder is intentionally using a host -# LLVM, rather than the typical src/llvm-project LLVM. -ENV NO_DOWNLOAD_CI_LLVM 1 - -# Using llvm-link-shared due to libffi issues -- see #34486 -ENV RUST_CONFIGURE_ARGS \ - --build=x86_64-unknown-linux-gnu \ - --llvm-root=/usr/lib/llvm-13 \ - --enable-llvm-link-shared \ - --set rust.thin-lto-import-instr-limit=10 - -# NOTE: intentionally uses all of `x.py`, `x`, and `x.ps1` to make sure they all work on Linux. -ENV SCRIPT ../x.py --stage 2 test --exclude src/tools/tidy && \ - # Run the `mir-opt` tests again but this time for a 32-bit target. - # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have - # both 32-bit and 64-bit outputs updated by the PR author, before - # the PR is approved and tested for merging. - # It will also detect tests lacking `// EMIT_MIR_FOR_EACH_BIT_WIDTH`, - # despite having different output on 32-bit vs 64-bit targets. - ../x --stage 2 test tests/mir-opt \ - --host='' --target=i686-unknown-linux-gnu && \ - # Run the UI test suite again, but in `--pass=check` mode - # - # This is intended to make sure that both `--pass=check` continues to - # work. - # - ../x.ps1 --stage 2 test tests/ui --pass=check \ - --host='' --target=i686-unknown-linux-gnu && \ - # Run tidy at the very end, after all the other tests. - python2.7 ../x.py --stage 2 test src/tools/tidy diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-14-stage1/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-14-stage1/Dockerfile new file mode 100644 index 000000000..b99a0886b --- /dev/null +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-14-stage1/Dockerfile @@ -0,0 +1,49 @@ +FROM ubuntu:22.04 + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + g++ \ + gcc-multilib \ + make \ + ninja-build \ + file \ + curl \ + ca-certificates \ + python3 \ + git \ + cmake \ + sudo \ + gdb \ + llvm-14-tools \ + llvm-14-dev \ + libedit-dev \ + libssl-dev \ + pkg-config \ + zlib1g-dev \ + xz-utils \ + nodejs \ + && rm -rf /var/lib/apt/lists/* + +COPY scripts/sccache.sh /scripts/ +RUN sh /scripts/sccache.sh + +# We are disabling CI LLVM since this builder is intentionally using a host +# LLVM, rather than the typical src/llvm-project LLVM. +ENV NO_DOWNLOAD_CI_LLVM 1 + +# Using llvm-link-shared due to libffi issues -- see #34486 +ENV RUST_CONFIGURE_ARGS \ + --build=x86_64-unknown-linux-gnu \ + --llvm-root=/usr/lib/llvm-14 \ + --enable-llvm-link-shared \ + --set rust.thin-lto-import-instr-limit=10 + +ENV SCRIPT ../x.py --stage 1 test --exclude src/tools/tidy && \ + # Run the `mir-opt` tests again but this time for a 32-bit target. + # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have + # both 32-bit and 64-bit outputs updated by the PR author, before + # the PR is approved and tested for merging. + # It will also detect tests lacking `// EMIT_MIR_FOR_EACH_BIT_WIDTH`, + # despite having different output on 32-bit vs 64-bit targets. + ../x.py --stage 1 test tests/mir-opt \ + --host='' --target=i686-unknown-linux-gnu diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-14/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-14/Dockerfile new file mode 100644 index 000000000..db6032f87 --- /dev/null +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-14/Dockerfile @@ -0,0 +1,67 @@ +FROM ubuntu:22.04 + +ARG DEBIAN_FRONTEND=noninteractive + +# NOTE: intentionally installs both python2 and python3 so we can test support for both. +RUN apt-get update && apt-get install -y --no-install-recommends \ + g++ \ + gcc-multilib \ + make \ + ninja-build \ + file \ + curl \ + ca-certificates \ + python2.7 \ + python3 \ + git \ + cmake \ + sudo \ + gdb \ + llvm-14-tools \ + llvm-14-dev \ + libedit-dev \ + libssl-dev \ + pkg-config \ + zlib1g-dev \ + xz-utils \ + nodejs \ + && rm -rf /var/lib/apt/lists/* + +# Install powershell (universal package) so we can test x.ps1 on Linux +RUN curl -sL "https://github.com/PowerShell/PowerShell/releases/download/v7.3.1/powershell_7.3.1-1.deb_amd64.deb" > powershell.deb && \ + dpkg -i powershell.deb && \ + rm -f powershell.deb + +COPY scripts/sccache.sh /scripts/ +RUN sh /scripts/sccache.sh + +# We are disabling CI LLVM since this builder is intentionally using a host +# LLVM, rather than the typical src/llvm-project LLVM. +ENV NO_DOWNLOAD_CI_LLVM 1 + +# Using llvm-link-shared due to libffi issues -- see #34486 +ENV RUST_CONFIGURE_ARGS \ + --build=x86_64-unknown-linux-gnu \ + --llvm-root=/usr/lib/llvm-14 \ + --enable-llvm-link-shared \ + --set rust.thin-lto-import-instr-limit=10 + +# NOTE: intentionally uses all of `x.py`, `x`, and `x.ps1` to make sure they all work on Linux. +ENV SCRIPT ../x.py --stage 2 test --exclude src/tools/tidy && \ + # Run the `mir-opt` tests again but this time for a 32-bit target. + # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have + # both 32-bit and 64-bit outputs updated by the PR author, before + # the PR is approved and tested for merging. + # It will also detect tests lacking `// EMIT_MIR_FOR_EACH_BIT_WIDTH`, + # despite having different output on 32-bit vs 64-bit targets. + ../x --stage 2 test tests/mir-opt \ + --host='' --target=i686-unknown-linux-gnu && \ + # Run the UI test suite again, but in `--pass=check` mode + # + # This is intended to make sure that both `--pass=check` continues to + # work. + # + ../x.ps1 --stage 2 test tests/ui --pass=check \ + --host='' --target=i686-unknown-linux-gnu && \ + # Run tidy at the very end, after all the other tests. + python2.7 ../x.py --stage 2 test src/tools/tidy diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile new file mode 100644 index 000000000..5219247cc --- /dev/null +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile @@ -0,0 +1,67 @@ +FROM ubuntu:22.10 + +ARG DEBIAN_FRONTEND=noninteractive + +# NOTE: intentionally installs both python2 and python3 so we can test support for both. +RUN apt-get update && apt-get install -y --no-install-recommends \ + g++ \ + gcc-multilib \ + make \ + ninja-build \ + file \ + curl \ + ca-certificates \ + python2.7 \ + python3 \ + git \ + cmake \ + sudo \ + gdb \ + llvm-15-tools \ + llvm-15-dev \ + libedit-dev \ + libssl-dev \ + pkg-config \ + zlib1g-dev \ + xz-utils \ + nodejs \ + && rm -rf /var/lib/apt/lists/* + +# Install powershell (universal package) so we can test x.ps1 on Linux +RUN curl -sL "https://github.com/PowerShell/PowerShell/releases/download/v7.3.1/powershell_7.3.1-1.deb_amd64.deb" > powershell.deb && \ + dpkg -i powershell.deb && \ + rm -f powershell.deb + +COPY scripts/sccache.sh /scripts/ +RUN sh /scripts/sccache.sh + +# We are disabling CI LLVM since this builder is intentionally using a host +# LLVM, rather than the typical src/llvm-project LLVM. +ENV NO_DOWNLOAD_CI_LLVM 1 + +# Using llvm-link-shared due to libffi issues -- see #34486 +ENV RUST_CONFIGURE_ARGS \ + --build=x86_64-unknown-linux-gnu \ + --llvm-root=/usr/lib/llvm-15 \ + --enable-llvm-link-shared \ + --set rust.thin-lto-import-instr-limit=10 + +# NOTE: intentionally uses all of `x.py`, `x`, and `x.ps1` to make sure they all work on Linux. +ENV SCRIPT ../x.py --stage 2 test --exclude src/tools/tidy && \ + # Run the `mir-opt` tests again but this time for a 32-bit target. + # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have + # both 32-bit and 64-bit outputs updated by the PR author, before + # the PR is approved and tested for merging. + # It will also detect tests lacking `// EMIT_MIR_FOR_EACH_BIT_WIDTH`, + # despite having different output on 32-bit vs 64-bit targets. + ../x --stage 2 test tests/mir-opt \ + --host='' --target=i686-unknown-linux-gnu && \ + # Run the UI test suite again, but in `--pass=check` mode + # + # This is intended to make sure that both `--pass=check` continues to + # work. + # + ../x.ps1 --stage 2 test tests/ui --pass=check \ + --host='' --target=i686-unknown-linux-gnu && \ + # Run tidy at the very end, after all the other tests. + python2.7 ../x.py --stage 2 test src/tools/tidy diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version b/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version index c39e9c5fb..9c550b2d7 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version @@ -1 +1 @@ -0.14.1 \ No newline at end of file +0.14.4 \ No newline at end of file diff --git a/src/ci/docker/scripts/fuchsia-test-runner.py b/src/ci/docker/scripts/fuchsia-test-runner.py index c8d1ff9ae..e7d1d9781 100755 --- a/src/ci/docker/scripts/fuchsia-test-runner.py +++ b/src/ci/docker/scripts/fuchsia-test-runner.py @@ -9,6 +9,7 @@ https://doc.rust-lang.org/stable/rustc/platform-support/fuchsia.html#aarch64-unk import argparse from dataclasses import dataclass +import fcntl import glob import hashlib import json @@ -19,18 +20,18 @@ import shutil import signal import subprocess import sys -from typing import ClassVar, List +from typing import ClassVar, List, Optional @dataclass class TestEnvironment: rust_dir: str sdk_dir: str - target_arch: str - package_server_pid: int = None - emu_addr: str = None - libstd_name: str = None - libtest_name: str = None + target: str + package_server_pid: Optional[int] = None + emu_addr: Optional[str] = None + libstd_name: Optional[str] = None + libtest_name: Optional[str] = None verbose: bool = False @staticmethod @@ -40,6 +41,15 @@ class TestEnvironment: return os.path.abspath(tmp_dir) return os.path.join(os.path.dirname(__file__), "tmp~") + @staticmethod + def triple_to_arch(triple): + if "x86_64" in triple: + return "x64" + elif "aarch64" in triple: + return "arm64" + else: + raise Exception(f"Unrecognized target triple {triple}") + @classmethod def env_file_path(cls): return os.path.join(cls.tmp_dir(), "test_env.json") @@ -49,7 +59,7 @@ class TestEnvironment: return cls( os.path.abspath(args.rust), os.path.abspath(args.sdk), - args.target_arch, + args.target, verbose=args.verbose, ) @@ -60,7 +70,7 @@ class TestEnvironment: return cls( test_env["rust_dir"], test_env["sdk_dir"], - test_env["target_arch"], + test_env["target"], libstd_name=test_env["libstd_name"], libtest_name=test_env["libtest_name"], emu_addr=test_env["emu_addr"], @@ -68,13 +78,6 @@ class TestEnvironment: verbose=test_env["verbose"], ) - def image_name(self): - if self.target_arch == "x64": - return "qemu-x64" - if self.target_arch == "arm64": - return "qemu-arm64" - raise Exception(f"Unrecognized target architecture {self.target_arch}") - def write_to_file(self): with open(self.env_file_path(), "w", encoding="utf-8") as f: f.write(json.dumps(self.__dict__)) @@ -108,13 +111,6 @@ class TestEnvironment: def repo_dir(self): return os.path.join(self.tmp_dir(), self.TEST_REPO_NAME) - def rustlib_dir(self): - if self.target_arch == "x64": - return "x86_64-unknown-fuchsia" - if self.target_arch == "arm64": - return "aarch64-unknown-fuchsia" - raise Exception(f"Unrecognized target architecture {self.target_arch}") - def libs_dir(self): return os.path.join( self.rust_dir, @@ -125,7 +121,7 @@ class TestEnvironment: return os.path.join( self.libs_dir(), "rustlib", - self.rustlib_dir(), + self.target, "lib", ) @@ -151,6 +147,9 @@ class TestEnvironment: def zxdb_script_path(self): return os.path.join(self.tmp_dir(), "zxdb_script") + def pm_lockfile_path(self): + return os.path.join(self.tmp_dir(), "pm.lock") + def log_info(self, msg): print(msg) @@ -384,7 +383,7 @@ class TestEnvironment: "--emulator-log", self.emulator_log_path(), "--image-name", - self.image_name(), + "qemu-" + self.triple_to_arch(self.target), ], stdout=self.subprocess_output(), stderr=self.subprocess_output(), @@ -465,6 +464,9 @@ class TestEnvironment: stderr=self.subprocess_output(), ) + # Create lockfiles + open(self.pm_lockfile_path(), 'a').close() + # Write to file self.write_to_file() @@ -512,9 +514,8 @@ class TestEnvironment: bin/{exe_name}={bin_path} lib/{libstd_name}={rust_dir}/lib/rustlib/{rustlib_dir}/lib/{libstd_name} lib/{libtest_name}={rust_dir}/lib/rustlib/{rustlib_dir}/lib/{libtest_name} - lib/ld.so.1={sdk_dir}/arch/{target_arch}/sysroot/lib/libc.so - lib/libzircon.so={sdk_dir}/arch/{target_arch}/sysroot/lib/libzircon.so - lib/libfdio.so={sdk_dir}/arch/{target_arch}/lib/libfdio.so + lib/ld.so.1={sdk_dir}/arch/{target_arch}/sysroot/dist/lib/ld.so.1 + lib/libfdio.so={sdk_dir}/arch/{target_arch}/dist/libfdio.so """ TEST_ENV_VARS: ClassVar[List[str]] = [ @@ -642,11 +643,11 @@ class TestEnvironment: package_dir=package_dir, package_name=package_name, rust_dir=self.rust_dir, - rustlib_dir=self.rustlib_dir(), + rustlib_dir=self.target, sdk_dir=self.sdk_dir, libstd_name=self.libstd_name, libtest_name=self.libtest_name, - target_arch=self.target_arch, + target_arch=self.triple_to_arch(self.target), ) ) for shared_lib in shared_libs: @@ -682,19 +683,25 @@ class TestEnvironment: log("Publishing package to repo...") # Publish package to repo - subprocess.check_call( - [ - self.tool_path("pm"), - "publish", - "-a", - "-repo", - self.repo_dir(), - "-f", - far_path, - ], - stdout=log_file, - stderr=log_file, - ) + with open(self.pm_lockfile_path(), 'w') as pm_lockfile: + fcntl.lockf(pm_lockfile.fileno(), fcntl.LOCK_EX) + subprocess.check_call( + [ + self.tool_path("pm"), + "publish", + "-a", + "-repo", + self.repo_dir(), + "-f", + far_path, + ], + stdout=log_file, + stderr=log_file, + ) + # This lock should be released automatically when the pm + # lockfile is closed, but we'll be polite and unlock it now + # since the spec leaves some wiggle room. + fcntl.lockf(pm_lockfile.fileno(), fcntl.LOCK_UN) log("Running ffx test...") @@ -849,23 +856,34 @@ class TestEnvironment: "--", "--build-id-dir", os.path.join(self.sdk_dir, ".build-id"), - "--build-id-dir", - os.path.join(self.libs_dir(), ".build-id"), ] - # Add rust source if it's available - if args.rust_src is not None: + libs_build_id_path = os.path.join(self.libs_dir(), ".build-id") + if os.path.exists(libs_build_id_path): + # Add .build-id symbols if installed libs have been stripped into a + # .build-id directory command += [ - "--build-dir", - args.rust_src, + "--build-id-dir", + libs_build_id_path, + ] + else: + # If no .build-id directory is detected, then assume that the shared + # libs contain their debug symbols + command += [ + f"--symbol-path={self.rust_dir}/lib/rustlib/{self.target}/lib", ] + # Add rust source if it's available + rust_src_map = None + if args.rust_src is not None: + # This matches the remapped prefix used by compiletest. There's no + # clear way that we can determine this, so it's hard coded. + rust_src_map = f"/rustc/FAKE_PREFIX={args.rust_src}" + # Add fuchsia source if it's available + fuchsia_src_map = None if args.fuchsia_src is not None: - command += [ - "--build-dir", - os.path.join(args.fuchsia_src, "out", "default"), - ] + fuchsia_src_map = f"./../..={args.fuchsia_src}" # Load debug symbols for the test binary and automatically attach if args.test is not None: @@ -888,7 +906,28 @@ class TestEnvironment: test_name, ) + # The fake-test-src-base directory maps to the suite directory + # e.g. tests/ui/foo.rs has a path of rust/fake-test-src-base/foo.rs + fake_test_src_base = os.path.join( + args.rust_src, + "fake-test-src-base", + ) + real_test_src_base = os.path.join( + args.rust_src, + "tests", + args.test.split(os.path.sep)[0], + ) + test_src_map = f"{fake_test_src_base}={real_test_src_base}" + with open(self.zxdb_script_path(), mode="w", encoding="utf-8") as f: + print(f"set source-map += {test_src_map}", file=f) + + if rust_src_map is not None: + print(f"set source-map += {rust_src_map}", file=f) + + if fuchsia_src_map is not None: + print(f"set source-map += {fuchsia_src_map}", file=f) + print(f"attach {test_name[:31]}", file=f) command += [ @@ -905,6 +944,20 @@ class TestEnvironment: # Connect to the running emulator with zxdb subprocess.run(command, env=self.ffx_cmd_env(), check=False) + def syslog(self, args): + subprocess.run( + [ + self.tool_path("ffx"), + "--config", + self.ffx_user_config_path(), + "log", + "--since", + "now", + ], + env=self.ffx_cmd_env(), + check=False, + ) + def start(args): test_env = TestEnvironment.from_args(args) @@ -938,6 +991,12 @@ def debug(args): return 0 +def syslog(args): + test_env = TestEnvironment.read_from_file() + test_env.syslog(args) + return 0 + + def main(): parser = argparse.ArgumentParser() @@ -969,8 +1028,8 @@ def main(): action="store_true", ) start_parser.add_argument( - "--target-arch", - help="the architecture of the image to test", + "--target", + help="the target platform to test", required=True, ) start_parser.set_defaults(func=start) @@ -1033,6 +1092,11 @@ def main(): ) debug_parser.set_defaults(func=debug) + syslog_parser = subparsers.add_parser( + "syslog", help="prints the device syslog" + ) + syslog_parser.set_defaults(func=syslog) + args = parser.parse_args() return args.func(args) diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 743b57ed4..c594288dc 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -33,6 +33,7 @@ x--expand-yaml-anchors--remove: - &shared-ci-variables CI_JOB_NAME: ${{ matrix.name }} + CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse - &public-variables SCCACHE_BUCKET: rust-lang-ci-sccache2 @@ -73,15 +74,15 @@ x--expand-yaml-anchors--remove: env: {} - &job-linux-xl - os: ubuntu-20.04-xl + os: ubuntu-20.04-16core-64gb <<: *base-job - &job-macos-xl - os: macos-latest # We don't have an XL builder for this + os: macos-12-xl <<: *base-job - &job-windows-xl - os: windows-latest-xl + os: windows-2019-8core-32gb <<: *base-job - &job-aarch64-linux @@ -299,15 +300,13 @@ jobs: <<: *job-linux-xl tidy: true - - name: x86_64-gnu-llvm-13 + - name: x86_64-gnu-llvm-14 <<: *job-linux-xl tidy: false - name: x86_64-gnu-tools <<: *job-linux-xl tidy: false - env: - CI_ONLY_WHEN_SUBMODULES_CHANGED: 1 auto: permissions: @@ -450,12 +449,17 @@ jobs: - name: x86_64-gnu-distcheck <<: *job-linux-xl - - name: x86_64-gnu-llvm-13 + - name: x86_64-gnu-llvm-15 + env: + RUST_BACKTRACE: 1 + <<: *job-linux-xl + + - name: x86_64-gnu-llvm-14 env: RUST_BACKTRACE: 1 <<: *job-linux-xl - - name: x86_64-gnu-llvm-13-stage1 + - name: x86_64-gnu-llvm-14-stage1 env: RUST_BACKTRACE: 1 <<: *job-linux-xl @@ -619,9 +623,7 @@ jobs: - name: i686-mingw-1 env: - RUST_CONFIGURE_ARGS: >- - --build=i686-pc-windows-gnu - --set llvm.allow-old-toolchain + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu SCRIPT: make ci-mingw-subset-1 # We are intentionally allowing an old toolchain on this builder (and that's # incompatible with LLVM downloads today). @@ -631,9 +633,7 @@ jobs: - name: i686-mingw-2 env: - RUST_CONFIGURE_ARGS: >- - --build=i686-pc-windows-gnu - --set llvm.allow-old-toolchain + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu SCRIPT: make ci-mingw-subset-2 # We are intentionally allowing an old toolchain on this builder (and that's # incompatible with LLVM downloads today). @@ -647,7 +647,6 @@ jobs: RUST_CONFIGURE_ARGS: >- --build=x86_64-pc-windows-gnu --enable-profiler - --set llvm.allow-old-toolchain # We are intentionally allowing an old toolchain on this builder (and that's # incompatible with LLVM downloads today). NO_DOWNLOAD_CI_LLVM: 1 @@ -660,7 +659,6 @@ jobs: RUST_CONFIGURE_ARGS: >- --build=x86_64-pc-windows-gnu --enable-profiler - --set llvm.allow-old-toolchain # We are intentionally allowing an old toolchain on this builder (and that's # incompatible with LLVM downloads today). NO_DOWNLOAD_CI_LLVM: 1 @@ -675,7 +673,7 @@ jobs: --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler - SCRIPT: PGO_HOST=x86_64-pc-windows-msvc src/ci/pgo.sh python x.py dist bootstrap --include-default-paths + SCRIPT: PGO_HOST=x86_64-pc-windows-msvc python src/ci/stage-build.py python x.py dist bootstrap --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-xl @@ -711,7 +709,6 @@ jobs: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler - --set llvm.allow-old-toolchain # We are intentionally allowing an old toolchain on this builder (and that's # incompatible with LLVM downloads today). NO_DOWNLOAD_CI_LLVM: 1 @@ -727,7 +724,6 @@ jobs: --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler - --set llvm.allow-old-toolchain # We are intentionally allowing an old toolchain on this builder (and that's # incompatible with LLVM downloads today). NO_DOWNLOAD_CI_LLVM: 1 diff --git a/src/ci/pgo.sh b/src/ci/pgo.sh deleted file mode 100755 index cbe32920a..000000000 --- a/src/ci/pgo.sh +++ /dev/null @@ -1,230 +0,0 @@ -#!/bin/bash -# ignore-tidy-linelength - -set -euxo pipefail - -ci_dir=`cd $(dirname $0) && pwd` -source "$ci_dir/shared.sh" - -# The root checkout, where the source is located -CHECKOUT=/checkout - -DOWNLOADED_LLVM=/rustroot - -# The main directory where the build occurs, which can be different between linux and windows -BUILD_ROOT=$CHECKOUT/obj - -if isWindows; then - CHECKOUT=$(pwd) - DOWNLOADED_LLVM=$CHECKOUT/citools/clang-rust - BUILD_ROOT=$CHECKOUT -fi - -# The various build artifacts used in other commands: to launch rustc builds, build the perf -# collector, and run benchmarks to gather profiling data -BUILD_ARTIFACTS=$BUILD_ROOT/build/$PGO_HOST -RUSTC_STAGE_0=$BUILD_ARTIFACTS/stage0/bin/rustc -CARGO_STAGE_0=$BUILD_ARTIFACTS/stage0/bin/cargo -RUSTC_STAGE_2=$BUILD_ARTIFACTS/stage2/bin/rustc - -# Windows needs these to have the .exe extension -if isWindows; then - RUSTC_STAGE_0="${RUSTC_STAGE_0}.exe" - CARGO_STAGE_0="${CARGO_STAGE_0}.exe" - RUSTC_STAGE_2="${RUSTC_STAGE_2}.exe" -fi - -# Make sure we have a temporary PGO work folder -PGO_TMP=/tmp/tmp-pgo -mkdir -p $PGO_TMP -rm -rf $PGO_TMP/* - -RUSTC_PERF=$PGO_TMP/rustc-perf - -# Compile several crates to gather execution PGO profiles. -# Arg0 => profiles (Debug, Opt) -# Arg1 => scenarios (Full, IncrFull, All) -# Arg2 => crates (syn, cargo, ...) -gather_profiles () { - cd $BUILD_ROOT - - # Compile libcore, both in opt-level=0 and opt-level=3 - RUSTC_BOOTSTRAP=1 $RUSTC_STAGE_2 \ - --edition=2021 --crate-type=lib $CHECKOUT/library/core/src/lib.rs \ - --out-dir $PGO_TMP - RUSTC_BOOTSTRAP=1 $RUSTC_STAGE_2 \ - --edition=2021 --crate-type=lib -Copt-level=3 $CHECKOUT/library/core/src/lib.rs \ - --out-dir $PGO_TMP - - cd $RUSTC_PERF - - # Run rustc-perf benchmarks - # Benchmark using profile_local with eprintln, which essentially just means - # don't actually benchmark -- just make sure we run rustc a bunch of times. - RUST_LOG=collector=debug \ - RUSTC=$RUSTC_STAGE_0 \ - RUSTC_BOOTSTRAP=1 \ - $CARGO_STAGE_0 run -p collector --bin collector -- \ - profile_local \ - eprintln \ - $RUSTC_STAGE_2 \ - --id Test \ - --profiles $1 \ - --cargo $CARGO_STAGE_0 \ - --scenarios $2 \ - --include $3 - - cd $BUILD_ROOT -} - -# This path has to be absolute -LLVM_PROFILE_DIRECTORY_ROOT=$PGO_TMP/llvm-pgo - -# We collect LLVM profiling information and rustc profiling information in -# separate phases. This increases build time -- though not by a huge amount -- -# but prevents any problems from arising due to different profiling runtimes -# being simultaneously linked in. -# LLVM IR PGO does not respect LLVM_PROFILE_FILE, so we have to set the profiling file -# path through our custom environment variable. We include the PID in the directory path -# to avoid updates to profile files being lost because of race conditions. -LLVM_PROFILE_DIR=${LLVM_PROFILE_DIRECTORY_ROOT}/prof-%p python3 $CHECKOUT/x.py build \ - --target=$PGO_HOST \ - --host=$PGO_HOST \ - --stage 2 library/std \ - --llvm-profile-generate - -# Compile rustc-perf: -# - get the expected commit source code: on linux, the Dockerfile downloads a source archive before -# running this script. On Windows, we do that here. -if isLinux; then - cp -r /tmp/rustc-perf $RUSTC_PERF - chown -R $(whoami): $RUSTC_PERF -else - # rustc-perf version from 2022-07-22 - PERF_COMMIT=3c253134664fdcba862c539d37f0de18557a9a4c - retry curl -LS -o $PGO_TMP/perf.zip \ - https://github.com/rust-lang/rustc-perf/archive/$PERF_COMMIT.zip && \ - cd $PGO_TMP && unzip -q perf.zip && \ - mv rustc-perf-$PERF_COMMIT $RUSTC_PERF && \ - rm perf.zip -fi - -# - build rustc-perf's collector ahead of time, which is needed to make sure the rustc-fake binary -# used by the collector is present. -cd $RUSTC_PERF - -RUSTC=$RUSTC_STAGE_0 \ -RUSTC_BOOTSTRAP=1 \ -$CARGO_STAGE_0 build -p collector - -# Here we're profiling LLVM, so we only care about `Debug` and `Opt`, because we want to stress -# codegen. We also profile some of the most prolific crates. -gather_profiles "Debug,Opt" "Full" \ - "syn-1.0.89,cargo-0.60.0,serde-1.0.136,ripgrep-13.0.0,regex-1.5.5,clap-3.1.6,hyper-0.14.18" - -LLVM_PROFILE_MERGED_FILE=$PGO_TMP/llvm-pgo.profdata - -# Merge the profile data we gathered for LLVM -# Note that this uses the profdata from the clang we used to build LLVM, -# which likely has a different version than our in-tree clang. -$DOWNLOADED_LLVM/bin/llvm-profdata merge -o ${LLVM_PROFILE_MERGED_FILE} ${LLVM_PROFILE_DIRECTORY_ROOT} - -echo "LLVM PGO statistics" -du -sh ${LLVM_PROFILE_MERGED_FILE} -du -sh ${LLVM_PROFILE_DIRECTORY_ROOT} -echo "Profile file count" -find ${LLVM_PROFILE_DIRECTORY_ROOT} -type f | wc -l - -# We don't need the individual .profraw files now that they have been merged into a final .profdata -rm -r $LLVM_PROFILE_DIRECTORY_ROOT - -# Rustbuild currently doesn't support rebuilding LLVM when PGO options -# change (or any other llvm-related options); so just clear out the relevant -# directories ourselves. -rm -r $BUILD_ARTIFACTS/llvm $BUILD_ARTIFACTS/lld - -# Okay, LLVM profiling is done, switch to rustc PGO. - -# The path has to be absolute -RUSTC_PROFILE_DIRECTORY_ROOT=$PGO_TMP/rustc-pgo - -python3 $CHECKOUT/x.py build --target=$PGO_HOST --host=$PGO_HOST \ - --stage 2 library/std \ - --rust-profile-generate=${RUSTC_PROFILE_DIRECTORY_ROOT} - -# Here we're profiling the `rustc` frontend, so we also include `Check`. -# The benchmark set includes various stress tests that put the frontend under pressure. -if isLinux; then - # The profile data is written into a single filepath that is being repeatedly merged when each - # rustc invocation ends. Empirically, this can result in some profiling data being lost. That's - # why we override the profile path to include the PID. This will produce many more profiling - # files, but the resulting profile will produce a slightly faster rustc binary. - LLVM_PROFILE_FILE=${RUSTC_PROFILE_DIRECTORY_ROOT}/default_%m_%p.profraw gather_profiles \ - "Check,Debug,Opt" "All" \ - "externs,ctfe-stress-5,cargo-0.60.0,token-stream-stress,match-stress,tuple-stress,diesel-1.4.8,bitmaps-3.1.0" -else - # On windows, we don't do that yet (because it generates a lot of data, hitting disk space - # limits on the builder), and use the default profraw merging behavior. - gather_profiles \ - "Check,Debug,Opt" "All" \ - "externs,ctfe-stress-5,cargo-0.60.0,token-stream-stress,match-stress,tuple-stress,diesel-1.4.8,bitmaps-3.1.0" -fi - -RUSTC_PROFILE_MERGED_FILE=$PGO_TMP/rustc-pgo.profdata - -# Merge the profile data we gathered -$BUILD_ARTIFACTS/llvm/bin/llvm-profdata \ - merge -o ${RUSTC_PROFILE_MERGED_FILE} ${RUSTC_PROFILE_DIRECTORY_ROOT} - -echo "Rustc PGO statistics" -du -sh ${RUSTC_PROFILE_MERGED_FILE} -du -sh ${RUSTC_PROFILE_DIRECTORY_ROOT} -echo "Profile file count" -find ${RUSTC_PROFILE_DIRECTORY_ROOT} -type f | wc -l - -# We don't need the individual .profraw files now that they have been merged into a final .profdata -rm -r $RUSTC_PROFILE_DIRECTORY_ROOT - -# Rustbuild currently doesn't support rebuilding LLVM when PGO options -# change (or any other llvm-related options); so just clear out the relevant -# directories ourselves. -rm -r $BUILD_ARTIFACTS/llvm $BUILD_ARTIFACTS/lld - -if isLinux; then - # Gather BOLT profile (BOLT is currently only available on Linux) - python3 ../x.py build --target=$PGO_HOST --host=$PGO_HOST \ - --stage 2 library/std \ - --llvm-profile-use=${LLVM_PROFILE_MERGED_FILE} \ - --llvm-bolt-profile-generate - - BOLT_PROFILE_MERGED_FILE=/tmp/bolt.profdata - - # Here we're profiling Bolt. - gather_profiles "Check,Debug,Opt" "Full" \ - "syn-1.0.89,serde-1.0.136,ripgrep-13.0.0,regex-1.5.5,clap-3.1.6,hyper-0.14.18" - - merge-fdata /tmp/prof.fdata* > ${BOLT_PROFILE_MERGED_FILE} - - echo "BOLT statistics" - du -sh /tmp/prof.fdata* - du -sh ${BOLT_PROFILE_MERGED_FILE} - echo "Profile file count" - find /tmp/prof.fdata* -type f | wc -l - - rm -r $BUILD_ARTIFACTS/llvm $BUILD_ARTIFACTS/lld - - # This produces the actual final set of artifacts, using both the LLVM and rustc - # collected profiling data. - $@ \ - --rust-profile-use=${RUSTC_PROFILE_MERGED_FILE} \ - --llvm-profile-use=${LLVM_PROFILE_MERGED_FILE} \ - --llvm-bolt-profile-use=${BOLT_PROFILE_MERGED_FILE} -else - $@ \ - --rust-profile-use=${RUSTC_PROFILE_MERGED_FILE} \ - --llvm-profile-use=${LLVM_PROFILE_MERGED_FILE} -fi - -echo "Rustc binary size" -ls -la ./build/$PGO_HOST/stage2/bin -ls -la ./build/$PGO_HOST/stage2/lib diff --git a/src/ci/run.sh b/src/ci/run.sh index 0db9c993e..efeb850cd 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -45,6 +45,8 @@ fi ci_dir=`cd $(dirname $0) && pwd` source "$ci_dir/shared.sh" +export CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse + if ! isCI || isCiBranch auto || isCiBranch beta || isCiBranch try || isCiBranch try-perf; then RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set build.print-step-timings --enable-verbose-tests" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set build.metrics" @@ -57,6 +59,14 @@ RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-locked-deps" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-cargo-native-static" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-units-std=1" +# When building for mingw, limit the number of parallel linker jobs during +# the LLVM build, as not to run out of memory. +# This is an attempt to fix the spurious build error tracked by +# https://github.com/rust-lang/rust/issues/108227. +if isWindows && [[ ${CUSTOM_MINGW-0} -eq 1 ]]; then + RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set llvm.link-jobs=1" +fi + # Only produce xz tarballs on CI. gz tarballs will be generated by the release # process by recompressing the existing xz ones. This decreases the storage # space required for CI artifacts. diff --git a/src/ci/scripts/install-mingw.sh b/src/ci/scripts/install-mingw.sh index 1685fbbbb..7eccb9b86 100755 --- a/src/ci/scripts/install-mingw.sh +++ b/src/ci/scripts/install-mingw.sh @@ -2,24 +2,6 @@ # If we need to download a custom MinGW, do so here and set the path # appropriately. # -# Here we also do a pretty heinous thing which is to mangle the MinGW -# installation we just downloaded. Currently, as of this writing, we're using -# MinGW-w64 builds of gcc, and that's currently at 6.3.0. We use 6.3.0 as it -# appears to be the first version which contains a fix for #40546, builds -# randomly failing during LLVM due to ar.exe/ranlib.exe failures. -# -# Unfortunately, though, 6.3.0 *also* is the first version of MinGW-w64 builds -# to contain a regression in gdb (#40184). As a result if we were to use the -# gdb provided (7.11.1) then we would fail all debuginfo tests. -# -# In order to fix spurious failures (pretty high priority) we use 6.3.0. To -# avoid disabling gdb tests we download an *old* version of gdb, specifically -# that found inside the 6.2.0 distribution. We then overwrite the 6.3.0 gdb -# with the 6.2.0 gdb to get tests passing. -# -# Note that we don't literally overwrite the gdb.exe binary because it appears -# to just use gdborig.exe, so that's the binary we deal with instead. -# # Otherwise install MinGW through `pacman` set -euo pipefail @@ -27,8 +9,8 @@ IFS=$'\n\t' source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" -MINGW_ARCHIVE_32="i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z" -MINGW_ARCHIVE_64="x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z" +MINGW_ARCHIVE_32="i686-12.2.0-release-posix-dwarf-rt_v10-rev0.7z" +MINGW_ARCHIVE_64="x86_64-12.2.0-release-posix-seh-rt_v10-rev0.7z" if isWindows; then case "${CI_JOB_NAME}" in @@ -66,7 +48,6 @@ if isWindows; then curl -o mingw.7z "${MIRRORS_BASE}/${mingw_archive}" 7z x -y mingw.7z > /dev/null - curl -o "${mingw_dir}/bin/gdborig.exe" "${MIRRORS_BASE}/2017-04-20-${bits}bit-gdborig.exe" ciCommandAddPath "$(pwd)/${mingw_dir}/bin" fi fi diff --git a/src/ci/scripts/should-skip-this.sh b/src/ci/scripts/should-skip-this.sh index 85d772253..48127166a 100755 --- a/src/ci/scripts/should-skip-this.sh +++ b/src/ci/scripts/should-skip-this.sh @@ -1,46 +1,11 @@ #!/bin/bash -# Set the SKIP_JOB environment variable if this job is supposed to only run -# when submodules are updated and they were not. The following time consuming -# tasks will be skipped when the environment variable is present. +# Set the SKIP_JOB environment variable if this job is not supposed to run on the current builder. set -euo pipefail IFS=$'\n\t' source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" -if [[ -n "${CI_ONLY_WHEN_SUBMODULES_CHANGED-}" ]]; then - git fetch "https://github.com/$GITHUB_REPOSITORY" "$GITHUB_BASE_REF" - BASE_COMMIT="$(git merge-base FETCH_HEAD HEAD)" - - echo "Searching for toolstate changes between $BASE_COMMIT and $(git rev-parse HEAD)" - - if git diff "$BASE_COMMIT" | grep --quiet "^index .* 160000"; then - # Submodules pseudo-files inside git have the 160000 permissions, so when - # those files are present in the diff a submodule was updated. - echo "Submodules were updated" - elif ! (git diff --quiet "$BASE_COMMIT" -- \ - src/tools/clippy src/tools/rustfmt src/tools/miri \ - library/std/src/sys); then - # There is not an easy blanket search for subtrees. For now, manually list - # the subtrees. - # Also run this when the platform-specific parts of std change, in case - # that breaks Miri. - echo "Tool subtrees were updated" - elif ! (git diff --quiet "$BASE_COMMIT" -- \ - tests/rustdoc-gui \ - src/librustdoc \ - src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile \ - src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version \ - src/tools/rustdoc-gui); then - # There was a change in either rustdoc or in its GUI tests. - echo "Rustdoc was updated" - else - echo "Not executing this job since no submodules nor subtrees were updated" - ciCommandSetEnv SKIP_JOB 1 - exit 0 - fi -fi - if [[ -n "${CI_ONLY_WHEN_CHANNEL-}" ]]; then if [[ "${CI_ONLY_WHEN_CHANNEL}" = "$(cat src/ci/channel)" ]]; then echo "The channel is the expected one" diff --git a/src/ci/stage-build.py b/src/ci/stage-build.py new file mode 100644 index 000000000..bd8fd524a --- /dev/null +++ b/src/ci/stage-build.py @@ -0,0 +1,842 @@ +#!/usr/bin/env python3 +# ignore-tidy-linelength + +# Compatible with Python 3.6+ + +import contextlib +import getpass +import glob +import json +import logging +import os +import pprint +import shutil +import subprocess +import sys +import time +import traceback +import urllib.request +from io import StringIO +from pathlib import Path +from typing import Callable, ContextManager, Dict, Iterable, Iterator, List, Optional, \ + Tuple, Union + +PGO_HOST = os.environ["PGO_HOST"] + +LOGGER = logging.getLogger("stage-build") + +LLVM_PGO_CRATES = [ + "syn-1.0.89", + "cargo-0.60.0", + "serde-1.0.136", + "ripgrep-13.0.0", + "regex-1.5.5", + "clap-3.1.6", + "hyper-0.14.18" +] + +RUSTC_PGO_CRATES = [ + "externs", + "ctfe-stress-5", + "cargo-0.60.0", + "token-stream-stress", + "match-stress", + "tuple-stress", + "diesel-1.4.8", + "bitmaps-3.1.0" +] + +LLVM_BOLT_CRATES = LLVM_PGO_CRATES + + +class Pipeline: + # Paths + def checkout_path(self) -> Path: + """ + The root checkout, where the source is located. + """ + raise NotImplementedError + + def downloaded_llvm_dir(self) -> Path: + """ + Directory where the host LLVM is located. + """ + raise NotImplementedError + + def build_root(self) -> Path: + """ + The main directory where the build occurs. + """ + raise NotImplementedError + + def build_artifacts(self) -> Path: + return self.build_root() / "build" / PGO_HOST + + def rustc_stage_0(self) -> Path: + return self.build_artifacts() / "stage0" / "bin" / "rustc" + + def cargo_stage_0(self) -> Path: + return self.build_artifacts() / "stage0" / "bin" / "cargo" + + def rustc_stage_2(self) -> Path: + return self.build_artifacts() / "stage2" / "bin" / "rustc" + + def opt_artifacts(self) -> Path: + raise NotImplementedError + + def llvm_profile_dir_root(self) -> Path: + return self.opt_artifacts() / "llvm-pgo" + + def llvm_profile_merged_file(self) -> Path: + return self.opt_artifacts() / "llvm-pgo.profdata" + + def rustc_perf_dir(self) -> Path: + return self.opt_artifacts() / "rustc-perf" + + def build_rustc_perf(self): + raise NotImplementedError() + + def rustc_profile_dir_root(self) -> Path: + return self.opt_artifacts() / "rustc-pgo" + + def rustc_profile_merged_file(self) -> Path: + return self.opt_artifacts() / "rustc-pgo.profdata" + + def rustc_profile_template_path(self) -> Path: + """ + The profile data is written into a single filepath that is being repeatedly merged when each + rustc invocation ends. Empirically, this can result in some profiling data being lost. That's + why we override the profile path to include the PID. This will produce many more profiling + files, but the resulting profile will produce a slightly faster rustc binary. + """ + return self.rustc_profile_dir_root() / "default_%m_%p.profraw" + + def supports_bolt(self) -> bool: + raise NotImplementedError + + def llvm_bolt_profile_merged_file(self) -> Path: + return self.opt_artifacts() / "bolt.profdata" + + def metrics_path(self) -> Path: + return self.build_root() / "build" / "metrics.json" + + +class LinuxPipeline(Pipeline): + def checkout_path(self) -> Path: + return Path("/checkout") + + def downloaded_llvm_dir(self) -> Path: + return Path("/rustroot") + + def build_root(self) -> Path: + return self.checkout_path() / "obj" + + def opt_artifacts(self) -> Path: + return Path("/tmp/tmp-multistage/opt-artifacts") + + def build_rustc_perf(self): + # /tmp/rustc-perf comes from the Dockerfile + shutil.copytree("/tmp/rustc-perf", self.rustc_perf_dir()) + cmd(["chown", "-R", f"{getpass.getuser()}:", self.rustc_perf_dir()]) + + with change_cwd(self.rustc_perf_dir()): + cmd([self.cargo_stage_0(), "build", "-p", "collector"], env=dict( + RUSTC=str(self.rustc_stage_0()), + RUSTC_BOOTSTRAP="1" + )) + + def supports_bolt(self) -> bool: + return True + + +class WindowsPipeline(Pipeline): + def __init__(self): + self.checkout_dir = Path(os.getcwd()) + + def checkout_path(self) -> Path: + return self.checkout_dir + + def downloaded_llvm_dir(self) -> Path: + return self.checkout_path() / "citools" / "clang-rust" + + def build_root(self) -> Path: + return self.checkout_path() + + def opt_artifacts(self) -> Path: + return self.checkout_path() / "opt-artifacts" + + def rustc_stage_0(self) -> Path: + return super().rustc_stage_0().with_suffix(".exe") + + def cargo_stage_0(self) -> Path: + return super().cargo_stage_0().with_suffix(".exe") + + def rustc_stage_2(self) -> Path: + return super().rustc_stage_2().with_suffix(".exe") + + def build_rustc_perf(self): + # rustc-perf version from 2022-07-22 + perf_commit = "3c253134664fdcba862c539d37f0de18557a9a4c" + rustc_perf_zip_path = self.opt_artifacts() / "perf.zip" + + def download_rustc_perf(): + download_file( + f"https://github.com/rust-lang/rustc-perf/archive/{perf_commit}.zip", + rustc_perf_zip_path + ) + with change_cwd(self.opt_artifacts()): + unpack_archive(rustc_perf_zip_path) + move_path(Path(f"rustc-perf-{perf_commit}"), self.rustc_perf_dir()) + delete_file(rustc_perf_zip_path) + + retry_action(download_rustc_perf, "Download rustc-perf") + + with change_cwd(self.rustc_perf_dir()): + cmd([self.cargo_stage_0(), "build", "-p", "collector"], env=dict( + RUSTC=str(self.rustc_stage_0()), + RUSTC_BOOTSTRAP="1" + )) + + def rustc_profile_template_path(self) -> Path: + """ + On Windows, we don't have enough space to use separate files for each rustc invocation. + Therefore, we use a single file for the generated profiles. + """ + return self.rustc_profile_dir_root() / "default_%m.profraw" + + def supports_bolt(self) -> bool: + return False + + +def get_timestamp() -> float: + return time.time() + + +Duration = float + + +def iterate_timers(timer: "Timer", name: str, level: int = 0) -> Iterator[ + Tuple[int, str, Duration]]: + """ + Hierarchically iterate the children of a timer, in a depth-first order. + """ + yield (level, name, timer.total_duration()) + for (child_name, child_timer) in timer.children: + yield from iterate_timers(child_timer, child_name, level=level + 1) + + +class Timer: + def __init__(self, parent_names: Tuple[str, ...] = ()): + self.children: List[Tuple[str, Timer]] = [] + self.section_active = False + self.parent_names = parent_names + self.duration_excluding_children: Duration = 0 + + @contextlib.contextmanager + def section(self, name: str) -> ContextManager["Timer"]: + assert not self.section_active + self.section_active = True + + start = get_timestamp() + exc = None + + child_timer = Timer(parent_names=self.parent_names + (name,)) + full_name = " > ".join(child_timer.parent_names) + try: + LOGGER.info(f"Section `{full_name}` starts") + yield child_timer + except BaseException as exception: + exc = exception + raise + finally: + end = get_timestamp() + duration = end - start + + child_timer.duration_excluding_children = duration - child_timer.total_duration() + self.add_child(name, child_timer) + if exc is None: + LOGGER.info(f"Section `{full_name}` ended: OK ({duration:.2f}s)") + else: + LOGGER.info(f"Section `{full_name}` ended: FAIL ({duration:.2f}s)") + self.section_active = False + + def total_duration(self) -> Duration: + return self.duration_excluding_children + sum( + c.total_duration() for (_, c) in self.children) + + def has_children(self) -> bool: + return len(self.children) > 0 + + def print_stats(self): + rows = [] + for (child_name, child_timer) in self.children: + for (level, name, duration) in iterate_timers(child_timer, child_name, level=0): + label = f"{' ' * level}{name}:" + rows.append((label, duration)) + + # Empty row + rows.append(("", "")) + + total_duration_label = "Total duration:" + total_duration = self.total_duration() + rows.append((total_duration_label, humantime(total_duration))) + + space_after_label = 2 + max_label_length = max(16, max(len(label) for (label, _) in rows)) + space_after_label + + table_width = max_label_length + 23 + divider = "-" * table_width + + with StringIO() as output: + print(divider, file=output) + for (label, duration) in rows: + if isinstance(duration, Duration): + pct = (duration / total_duration) * 100 + value = f"{duration:>12.2f}s ({pct:>5.2f}%)" + else: + value = f"{duration:>{len(total_duration_label) + 7}}" + print(f"{label:<{max_label_length}} {value}", file=output) + print(divider, file=output, end="") + LOGGER.info(f"Timer results\n{output.getvalue()}") + + def add_child(self, name: str, timer: "Timer"): + self.children.append((name, timer)) + + def add_duration(self, name: str, duration: Duration): + timer = Timer(parent_names=self.parent_names + (name,)) + timer.duration_excluding_children = duration + self.add_child(name, timer) + + +class BuildStep: + def __init__(self, type: str, children: List["BuildStep"], duration: float): + self.type = type + self.children = children + self.duration = duration + + def find_all_by_type(self, type: str) -> Iterator["BuildStep"]: + if type == self.type: + yield self + for child in self.children: + yield from child.find_all_by_type(type) + + def __repr__(self): + return f"BuildStep(type={self.type}, duration={self.duration}, children={len(self.children)})" + + +def load_last_metrics(path: Path) -> BuildStep: + """ + Loads the metrics of the most recent bootstrap execution from a metrics.json file. + """ + with open(path, "r") as f: + metrics = json.load(f) + invocation = metrics["invocations"][-1] + + def parse(entry) -> Optional[BuildStep]: + if "kind" not in entry or entry["kind"] != "rustbuild_step": + return None + type = entry.get("type", "") + duration = entry.get("duration_excluding_children_sec", 0) + children = [] + + for child in entry.get("children", ()): + step = parse(child) + if step is not None: + children.append(step) + duration += step.duration + return BuildStep(type=type, children=children, duration=duration) + + children = [parse(child) for child in invocation.get("children", ())] + return BuildStep( + type="root", + children=children, + duration=invocation.get("duration_including_children_sec", 0) + ) + + +@contextlib.contextmanager +def change_cwd(dir: Path): + """ + Temporarily change working directory to `dir`. + """ + cwd = os.getcwd() + LOGGER.debug(f"Changing working dir from `{cwd}` to `{dir}`") + os.chdir(dir) + try: + yield + finally: + LOGGER.debug(f"Reverting working dir to `{cwd}`") + os.chdir(cwd) + + +def humantime(time_s: float) -> str: + hours = time_s // 3600 + time_s = time_s % 3600 + minutes = time_s // 60 + seconds = time_s % 60 + + result = "" + if hours > 0: + result += f"{int(hours)}h " + if minutes > 0: + result += f"{int(minutes)}m " + result += f"{round(seconds)}s" + return result + + +def move_path(src: Path, dst: Path): + LOGGER.info(f"Moving `{src}` to `{dst}`") + shutil.move(src, dst) + + +def delete_file(path: Path): + LOGGER.info(f"Deleting file `{path}`") + os.unlink(path) + + +def delete_directory(path: Path): + LOGGER.info(f"Deleting directory `{path}`") + shutil.rmtree(path) + + +def unpack_archive(archive: Path): + LOGGER.info(f"Unpacking archive `{archive}`") + shutil.unpack_archive(archive) + + +def download_file(src: str, target: Path): + LOGGER.info(f"Downloading `{src}` into `{target}`") + urllib.request.urlretrieve(src, str(target)) + + +def retry_action(action, name: str, max_fails: int = 5): + LOGGER.info(f"Attempting to perform action `{name}` with retry") + for iteration in range(max_fails): + LOGGER.info(f"Attempt {iteration + 1}/{max_fails}") + try: + action() + return + except: + LOGGER.error(f"Action `{name}` has failed\n{traceback.format_exc()}") + + raise Exception(f"Action `{name}` has failed after {max_fails} attempts") + + +def cmd( + args: List[Union[str, Path]], + env: Optional[Dict[str, str]] = None, + output_path: Optional[Path] = None +): + args = [str(arg) for arg in args] + + environment = os.environ.copy() + + cmd_str = "" + if env is not None: + environment.update(env) + cmd_str += " ".join(f"{k}={v}" for (k, v) in (env or {}).items()) + cmd_str += " " + cmd_str += " ".join(args) + if output_path is not None: + cmd_str += f" > {output_path}" + LOGGER.info(f"Executing `{cmd_str}`") + + if output_path is not None: + with open(output_path, "w") as f: + return subprocess.run( + args, + env=environment, + check=True, + stdout=f + ) + return subprocess.run(args, env=environment, check=True) + + +def run_compiler_benchmarks( + pipeline: Pipeline, + profiles: List[str], + scenarios: List[str], + crates: List[str], + env: Optional[Dict[str, str]] = None +): + env = env if env is not None else {} + + # Compile libcore, both in opt-level=0 and opt-level=3 + with change_cwd(pipeline.build_root()): + cmd([ + pipeline.rustc_stage_2(), + "--edition", "2021", + "--crate-type", "lib", + str(pipeline.checkout_path() / "library/core/src/lib.rs"), + "--out-dir", pipeline.opt_artifacts() + ], env=dict(RUSTC_BOOTSTRAP="1", **env)) + + cmd([ + pipeline.rustc_stage_2(), + "--edition", "2021", + "--crate-type", "lib", + "-Copt-level=3", + str(pipeline.checkout_path() / "library/core/src/lib.rs"), + "--out-dir", pipeline.opt_artifacts() + ], env=dict(RUSTC_BOOTSTRAP="1", **env)) + + # Run rustc-perf benchmarks + # Benchmark using profile_local with eprintln, which essentially just means + # don't actually benchmark -- just make sure we run rustc a bunch of times. + with change_cwd(pipeline.rustc_perf_dir()): + cmd([ + pipeline.cargo_stage_0(), + "run", + "-p", "collector", "--bin", "collector", "--", + "profile_local", "eprintln", + pipeline.rustc_stage_2(), + "--id", "Test", + "--cargo", pipeline.cargo_stage_0(), + "--profiles", ",".join(profiles), + "--scenarios", ",".join(scenarios), + "--include", ",".join(crates) + ], env=dict( + RUST_LOG="collector=debug", + RUSTC=str(pipeline.rustc_stage_0()), + RUSTC_BOOTSTRAP="1", + **env + )) + + +# https://stackoverflow.com/a/31631711/1107768 +def format_bytes(size: int) -> str: + """Return the given bytes as a human friendly KiB, MiB or GiB string.""" + KB = 1024 + MB = KB ** 2 # 1,048,576 + GB = KB ** 3 # 1,073,741,824 + TB = KB ** 4 # 1,099,511,627,776 + + if size < KB: + return f"{size} B" + elif KB <= size < MB: + return f"{size / KB:.2f} KiB" + elif MB <= size < GB: + return f"{size / MB:.2f} MiB" + elif GB <= size < TB: + return f"{size / GB:.2f} GiB" + else: + return str(size) + + +# https://stackoverflow.com/a/63307131/1107768 +def count_files(path: Path) -> int: + return sum(1 for p in path.rglob("*") if p.is_file()) + + +def count_files_with_prefix(path: Path) -> int: + return sum(1 for p in glob.glob(f"{path}*") if Path(p).is_file()) + + +# https://stackoverflow.com/a/55659577/1107768 +def get_path_size(path: Path) -> int: + if path.is_dir(): + return sum(p.stat().st_size for p in path.rglob("*")) + return path.stat().st_size + + +def get_path_prefix_size(path: Path) -> int: + """ + Get size of all files beginning with the prefix `path`. + Alternative to shell `du -sh *`. + """ + return sum(Path(p).stat().st_size for p in glob.glob(f"{path}*")) + + +def get_files(directory: Path, filter: Optional[Callable[[Path], bool]] = None) -> Iterable[Path]: + for file in os.listdir(directory): + path = directory / file + if filter is None or filter(path): + yield path + + +def build_rustc( + pipeline: Pipeline, + args: List[str], + env: Optional[Dict[str, str]] = None +): + arguments = [ + sys.executable, + pipeline.checkout_path() / "x.py", + "build", + "--target", PGO_HOST, + "--host", PGO_HOST, + "--stage", "2", + "library/std" + ] + args + cmd(arguments, env=env) + + +def create_pipeline() -> Pipeline: + if sys.platform == "linux": + return LinuxPipeline() + elif sys.platform in ("cygwin", "win32"): + return WindowsPipeline() + else: + raise Exception(f"Optimized build is not supported for platform {sys.platform}") + + +def gather_llvm_profiles(pipeline: Pipeline): + LOGGER.info("Running benchmarks with PGO instrumented LLVM") + run_compiler_benchmarks( + pipeline, + profiles=["Debug", "Opt"], + scenarios=["Full"], + crates=LLVM_PGO_CRATES + ) + + profile_path = pipeline.llvm_profile_merged_file() + LOGGER.info(f"Merging LLVM PGO profiles to {profile_path}") + cmd([ + pipeline.downloaded_llvm_dir() / "bin" / "llvm-profdata", + "merge", + "-o", profile_path, + pipeline.llvm_profile_dir_root() + ]) + + LOGGER.info("LLVM PGO statistics") + LOGGER.info(f"{profile_path}: {format_bytes(get_path_size(profile_path))}") + LOGGER.info( + f"{pipeline.llvm_profile_dir_root()}: {format_bytes(get_path_size(pipeline.llvm_profile_dir_root()))}") + LOGGER.info(f"Profile file count: {count_files(pipeline.llvm_profile_dir_root())}") + + # We don't need the individual .profraw files now that they have been merged + # into a final .profdata + delete_directory(pipeline.llvm_profile_dir_root()) + + +def gather_rustc_profiles(pipeline: Pipeline): + LOGGER.info("Running benchmarks with PGO instrumented rustc") + + # Here we're profiling the `rustc` frontend, so we also include `Check`. + # The benchmark set includes various stress tests that put the frontend under pressure. + run_compiler_benchmarks( + pipeline, + profiles=["Check", "Debug", "Opt"], + scenarios=["All"], + crates=RUSTC_PGO_CRATES, + env=dict( + LLVM_PROFILE_FILE=str(pipeline.rustc_profile_template_path()) + ) + ) + + profile_path = pipeline.rustc_profile_merged_file() + LOGGER.info(f"Merging Rustc PGO profiles to {profile_path}") + cmd([ + pipeline.build_artifacts() / "llvm" / "bin" / "llvm-profdata", + "merge", + "-o", profile_path, + pipeline.rustc_profile_dir_root() + ]) + + LOGGER.info("Rustc PGO statistics") + LOGGER.info(f"{profile_path}: {format_bytes(get_path_size(profile_path))}") + LOGGER.info( + f"{pipeline.rustc_profile_dir_root()}: {format_bytes(get_path_size(pipeline.rustc_profile_dir_root()))}") + LOGGER.info(f"Profile file count: {count_files(pipeline.rustc_profile_dir_root())}") + + # We don't need the individual .profraw files now that they have been merged + # into a final .profdata + delete_directory(pipeline.rustc_profile_dir_root()) + + +def gather_llvm_bolt_profiles(pipeline: Pipeline): + LOGGER.info("Running benchmarks with BOLT instrumented LLVM") + run_compiler_benchmarks( + pipeline, + profiles=["Check", "Debug", "Opt"], + scenarios=["Full"], + crates=LLVM_BOLT_CRATES + ) + + merged_profile_path = pipeline.llvm_bolt_profile_merged_file() + profile_files_path = Path("/tmp/prof.fdata") + LOGGER.info(f"Merging LLVM BOLT profiles to {merged_profile_path}") + + profile_files = sorted(glob.glob(f"{profile_files_path}*")) + cmd([ + "merge-fdata", + *profile_files, + ], output_path=merged_profile_path) + + LOGGER.info("LLVM BOLT statistics") + LOGGER.info(f"{merged_profile_path}: {format_bytes(get_path_size(merged_profile_path))}") + LOGGER.info( + f"{profile_files_path}: {format_bytes(get_path_prefix_size(profile_files_path))}") + LOGGER.info(f"Profile file count: {count_files_with_prefix(profile_files_path)}") + + +def clear_llvm_files(pipeline: Pipeline): + """ + Rustbuild currently doesn't support rebuilding LLVM when PGO options + change (or any other llvm-related options); so just clear out the relevant + directories ourselves. + """ + LOGGER.info("Clearing LLVM build files") + delete_directory(pipeline.build_artifacts() / "llvm") + delete_directory(pipeline.build_artifacts() / "lld") + + +def print_binary_sizes(pipeline: Pipeline): + bin_dir = pipeline.build_artifacts() / "stage2" / "bin" + binaries = get_files(bin_dir) + + lib_dir = pipeline.build_artifacts() / "stage2" / "lib" + libraries = get_files(lib_dir, lambda p: p.suffix == ".so") + + paths = sorted(binaries) + sorted(libraries) + with StringIO() as output: + for path in paths: + path_str = f"{path.name}:" + print(f"{path_str:<50}{format_bytes(path.stat().st_size):>14}", file=output) + LOGGER.info(f"Rustc binary size\n{output.getvalue()}") + + +def print_free_disk_space(pipeline: Pipeline): + usage = shutil.disk_usage(pipeline.opt_artifacts()) + total = usage.total + used = usage.used + free = usage.free + + logging.info( + f"Free disk space: {format_bytes(free)} out of total {format_bytes(total)} ({(used / total) * 100:.2f}% used)") + + +def log_metrics(step: BuildStep): + substeps: List[Tuple[int, BuildStep]] = [] + + def visit(step: BuildStep, level: int): + substeps.append((level, step)) + for child in step.children: + visit(child, level=level + 1) + + visit(step, 0) + + output = StringIO() + for (level, step) in substeps: + label = f"{'.' * level}{step.type}" + print(f"{label:<65}{step.duration:>8.2f}s", file=output) + logging.info(f"Build step durations\n{output.getvalue()}") + + +def record_metrics(pipeline: Pipeline, timer: Timer): + metrics = load_last_metrics(pipeline.metrics_path()) + if metrics is None: + return + llvm_steps = tuple(metrics.find_all_by_type("bootstrap::native::Llvm")) + assert len(llvm_steps) > 0 + llvm_duration = sum(step.duration for step in llvm_steps) + + rustc_steps = tuple(metrics.find_all_by_type("bootstrap::compile::Rustc")) + assert len(rustc_steps) > 0 + rustc_duration = sum(step.duration for step in rustc_steps) + + # The LLVM step is part of the Rustc step + rustc_duration -= llvm_duration + + timer.add_duration("LLVM", llvm_duration) + timer.add_duration("Rustc", rustc_duration) + + log_metrics(metrics) + + +def execute_build_pipeline(timer: Timer, pipeline: Pipeline, final_build_args: List[str]): + # Clear and prepare tmp directory + shutil.rmtree(pipeline.opt_artifacts(), ignore_errors=True) + os.makedirs(pipeline.opt_artifacts(), exist_ok=True) + + pipeline.build_rustc_perf() + + # Stage 1: Build rustc + PGO instrumented LLVM + with timer.section("Stage 1 (LLVM PGO)") as stage1: + with stage1.section("Build rustc and LLVM") as rustc_build: + build_rustc(pipeline, args=[ + "--llvm-profile-generate" + ], env=dict( + LLVM_PROFILE_DIR=str(pipeline.llvm_profile_dir_root() / "prof-%p") + )) + record_metrics(pipeline, rustc_build) + + with stage1.section("Gather profiles"): + gather_llvm_profiles(pipeline) + print_free_disk_space(pipeline) + + clear_llvm_files(pipeline) + final_build_args += [ + "--llvm-profile-use", + pipeline.llvm_profile_merged_file() + ] + + # Stage 2: Build PGO instrumented rustc + LLVM + with timer.section("Stage 2 (rustc PGO)") as stage2: + with stage2.section("Build rustc and LLVM") as rustc_build: + build_rustc(pipeline, args=[ + "--rust-profile-generate", + pipeline.rustc_profile_dir_root() + ]) + record_metrics(pipeline, rustc_build) + + with stage2.section("Gather profiles"): + gather_rustc_profiles(pipeline) + print_free_disk_space(pipeline) + + clear_llvm_files(pipeline) + final_build_args += [ + "--rust-profile-use", + pipeline.rustc_profile_merged_file() + ] + + # Stage 3: Build rustc + BOLT instrumented LLVM + if pipeline.supports_bolt(): + with timer.section("Stage 3 (LLVM BOLT)") as stage3: + with stage3.section("Build rustc and LLVM") as rustc_build: + build_rustc(pipeline, args=[ + "--llvm-profile-use", + pipeline.llvm_profile_merged_file(), + "--llvm-bolt-profile-generate", + ]) + record_metrics(pipeline, rustc_build) + + with stage3.section("Gather profiles"): + gather_llvm_bolt_profiles(pipeline) + + print_free_disk_space(pipeline) + clear_llvm_files(pipeline) + final_build_args += [ + "--llvm-bolt-profile-use", + pipeline.llvm_bolt_profile_merged_file() + ] + + # Stage 4: Build PGO optimized rustc + PGO/BOLT optimized LLVM + with timer.section("Stage 4 (final build)") as stage4: + cmd(final_build_args) + record_metrics(pipeline, stage4) + + +if __name__ == "__main__": + logging.basicConfig( + level=logging.DEBUG, + format="%(name)s %(levelname)-4s: %(message)s", + ) + + LOGGER.info(f"Running multi-stage build using Python {sys.version}") + LOGGER.info(f"Environment values\n{pprint.pformat(dict(os.environ), indent=2)}") + + build_args = sys.argv[1:] + + timer = Timer() + pipeline = create_pipeline() + try: + execute_build_pipeline(timer, pipeline, build_args) + except BaseException as e: + LOGGER.error("The multi-stage build has failed") + raise e + finally: + timer.print_stats() + print_free_disk_space(pipeline) + + print_binary_sizes(pipeline) -- cgit v1.2.3