From ef24de24a82fe681581cc130f342363c47c0969a Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 7 Jun 2024 07:48:48 +0200 Subject: Merging upstream version 1.75.0+dfsg1. Signed-off-by: Daniel Baumann --- src/bootstrap/CHANGELOG.md | 71 - src/bootstrap/Cargo.lock | 181 +- src/bootstrap/Cargo.toml | 39 +- src/bootstrap/README.md | 14 +- src/bootstrap/bin/_helper.rs | 24 - src/bootstrap/bin/main.rs | 124 - src/bootstrap/bin/rustc.rs | 410 --- src/bootstrap/bin/rustdoc.rs | 86 - src/bootstrap/bin/sccache-plus-cl.rs | 38 - src/bootstrap/bootstrap.py | 29 +- src/bootstrap/bootstrap_test.py | 3 +- src/bootstrap/builder.rs | 2318 -------------- src/bootstrap/builder/tests.rs | 702 ----- src/bootstrap/cache.rs | 272 -- src/bootstrap/cc_detect.rs | 273 -- src/bootstrap/channel.rs | 160 - src/bootstrap/check.rs | 542 ---- src/bootstrap/clean.rs | 237 -- src/bootstrap/compile.rs | 2015 ------------ src/bootstrap/config.rs | 2104 ------------- src/bootstrap/config/tests.rs | 198 -- src/bootstrap/configure.py | 20 +- src/bootstrap/defaults/config.codegen.toml | 2 +- src/bootstrap/dist.rs | 2274 -------------- src/bootstrap/doc.rs | 1085 ------- src/bootstrap/download-ci-llvm-stamp | 2 +- src/bootstrap/download.rs | 686 ---- src/bootstrap/dylib_util.rs | 30 - src/bootstrap/flags.rs | 565 ---- src/bootstrap/format.rs | 322 -- src/bootstrap/install.rs | 320 -- src/bootstrap/job.rs | 143 - src/bootstrap/lib.rs | 1846 ----------- src/bootstrap/llvm.rs | 1357 -------- src/bootstrap/metadata.rs | 101 - src/bootstrap/metrics.rs | 258 -- src/bootstrap/render_tests.rs | 400 --- src/bootstrap/run.rs | 297 -- src/bootstrap/sanity.rs | 267 -- src/bootstrap/setup.rs | 599 ---- src/bootstrap/setup/tests.rs | 14 - src/bootstrap/src/bin/main.rs | 143 + src/bootstrap/src/bin/rustc.rs | 406 +++ src/bootstrap/src/bin/rustdoc.rs | 92 + src/bootstrap/src/bin/sccache-plus-cl.rs | 38 + src/bootstrap/src/core/build_steps/check.rs | 544 ++++ src/bootstrap/src/core/build_steps/clean.rs | 242 ++ src/bootstrap/src/core/build_steps/compile.rs | 2069 ++++++++++++ src/bootstrap/src/core/build_steps/dist.rs | 2366 ++++++++++++++ src/bootstrap/src/core/build_steps/doc.rs | 1094 +++++++ src/bootstrap/src/core/build_steps/format.rs | 322 ++ src/bootstrap/src/core/build_steps/install.rs | 332 ++ src/bootstrap/src/core/build_steps/llvm.rs | 1366 ++++++++ src/bootstrap/src/core/build_steps/mod.rs | 15 + src/bootstrap/src/core/build_steps/run.rs | 301 ++ src/bootstrap/src/core/build_steps/setup.rs | 623 ++++ src/bootstrap/src/core/build_steps/suggest.rs | 78 + .../src/core/build_steps/synthetic_targets.rs | 82 + src/bootstrap/src/core/build_steps/test.rs | 3315 ++++++++++++++++++++ src/bootstrap/src/core/build_steps/tool.rs | 849 +++++ src/bootstrap/src/core/build_steps/toolstate.rs | 478 +++ src/bootstrap/src/core/builder.rs | 2392 ++++++++++++++ src/bootstrap/src/core/config/config.rs | 2208 +++++++++++++ src/bootstrap/src/core/config/flags.rs | 571 ++++ src/bootstrap/src/core/config/mod.rs | 4 + src/bootstrap/src/core/download.rs | 701 +++++ src/bootstrap/src/core/metadata.rs | 101 + src/bootstrap/src/core/mod.rs | 6 + src/bootstrap/src/core/sanity.rs | 267 ++ src/bootstrap/src/lib.rs | 1874 +++++++++++ src/bootstrap/src/tests/builder.rs | 701 +++++ src/bootstrap/src/tests/config.rs | 219 ++ src/bootstrap/src/tests/setup.rs | 14 + src/bootstrap/src/utils/bin_helpers.rs | 28 + src/bootstrap/src/utils/cache.rs | 272 ++ src/bootstrap/src/utils/cc_detect.rs | 288 ++ src/bootstrap/src/utils/channel.rs | 159 + src/bootstrap/src/utils/dylib.rs | 27 + src/bootstrap/src/utils/exec.rs | 60 + src/bootstrap/src/utils/helpers.rs | 472 +++ src/bootstrap/src/utils/job.rs | 159 + src/bootstrap/src/utils/metrics.rs | 258 ++ src/bootstrap/src/utils/mod.rs | 15 + src/bootstrap/src/utils/render_tests.rs | 400 +++ src/bootstrap/src/utils/tarball.rs | 381 +++ src/bootstrap/suggest.rs | 74 - src/bootstrap/synthetic_targets.rs | 82 - src/bootstrap/tarball.rs | 373 --- src/bootstrap/test.rs | 3093 ------------------ src/bootstrap/tool.rs | 848 ----- src/bootstrap/toolstate.rs | 478 --- src/bootstrap/util.rs | 497 --- 92 files changed, 26461 insertions(+), 25744 deletions(-) delete mode 100644 src/bootstrap/CHANGELOG.md delete mode 100644 src/bootstrap/bin/_helper.rs delete mode 100644 src/bootstrap/bin/main.rs delete mode 100644 src/bootstrap/bin/rustc.rs delete mode 100644 src/bootstrap/bin/rustdoc.rs delete mode 100644 src/bootstrap/bin/sccache-plus-cl.rs delete mode 100644 src/bootstrap/builder.rs delete mode 100644 src/bootstrap/builder/tests.rs delete mode 100644 src/bootstrap/cache.rs delete mode 100644 src/bootstrap/cc_detect.rs delete mode 100644 src/bootstrap/channel.rs delete mode 100644 src/bootstrap/check.rs delete mode 100644 src/bootstrap/clean.rs delete mode 100644 src/bootstrap/compile.rs delete mode 100644 src/bootstrap/config.rs delete mode 100644 src/bootstrap/config/tests.rs delete mode 100644 src/bootstrap/dist.rs delete mode 100644 src/bootstrap/doc.rs delete mode 100644 src/bootstrap/download.rs delete mode 100644 src/bootstrap/dylib_util.rs delete mode 100644 src/bootstrap/flags.rs delete mode 100644 src/bootstrap/format.rs delete mode 100644 src/bootstrap/install.rs delete mode 100644 src/bootstrap/job.rs delete mode 100644 src/bootstrap/lib.rs delete mode 100644 src/bootstrap/llvm.rs delete mode 100644 src/bootstrap/metadata.rs delete mode 100644 src/bootstrap/metrics.rs delete mode 100644 src/bootstrap/render_tests.rs delete mode 100644 src/bootstrap/run.rs delete mode 100644 src/bootstrap/sanity.rs delete mode 100644 src/bootstrap/setup.rs delete mode 100644 src/bootstrap/setup/tests.rs create mode 100644 src/bootstrap/src/bin/main.rs create mode 100644 src/bootstrap/src/bin/rustc.rs create mode 100644 src/bootstrap/src/bin/rustdoc.rs create mode 100644 src/bootstrap/src/bin/sccache-plus-cl.rs create mode 100644 src/bootstrap/src/core/build_steps/check.rs create mode 100644 src/bootstrap/src/core/build_steps/clean.rs create mode 100644 src/bootstrap/src/core/build_steps/compile.rs create mode 100644 src/bootstrap/src/core/build_steps/dist.rs create mode 100644 src/bootstrap/src/core/build_steps/doc.rs create mode 100644 src/bootstrap/src/core/build_steps/format.rs create mode 100644 src/bootstrap/src/core/build_steps/install.rs create mode 100644 src/bootstrap/src/core/build_steps/llvm.rs create mode 100644 src/bootstrap/src/core/build_steps/mod.rs create mode 100644 src/bootstrap/src/core/build_steps/run.rs create mode 100644 src/bootstrap/src/core/build_steps/setup.rs create mode 100644 src/bootstrap/src/core/build_steps/suggest.rs create mode 100644 src/bootstrap/src/core/build_steps/synthetic_targets.rs create mode 100644 src/bootstrap/src/core/build_steps/test.rs create mode 100644 src/bootstrap/src/core/build_steps/tool.rs create mode 100644 src/bootstrap/src/core/build_steps/toolstate.rs create mode 100644 src/bootstrap/src/core/builder.rs create mode 100644 src/bootstrap/src/core/config/config.rs create mode 100644 src/bootstrap/src/core/config/flags.rs create mode 100644 src/bootstrap/src/core/config/mod.rs create mode 100644 src/bootstrap/src/core/download.rs create mode 100644 src/bootstrap/src/core/metadata.rs create mode 100644 src/bootstrap/src/core/mod.rs create mode 100644 src/bootstrap/src/core/sanity.rs create mode 100644 src/bootstrap/src/lib.rs create mode 100644 src/bootstrap/src/tests/builder.rs create mode 100644 src/bootstrap/src/tests/config.rs create mode 100644 src/bootstrap/src/tests/setup.rs create mode 100644 src/bootstrap/src/utils/bin_helpers.rs create mode 100644 src/bootstrap/src/utils/cache.rs create mode 100644 src/bootstrap/src/utils/cc_detect.rs create mode 100644 src/bootstrap/src/utils/channel.rs create mode 100644 src/bootstrap/src/utils/dylib.rs create mode 100644 src/bootstrap/src/utils/exec.rs create mode 100644 src/bootstrap/src/utils/helpers.rs create mode 100644 src/bootstrap/src/utils/job.rs create mode 100644 src/bootstrap/src/utils/metrics.rs create mode 100644 src/bootstrap/src/utils/mod.rs create mode 100644 src/bootstrap/src/utils/render_tests.rs create mode 100644 src/bootstrap/src/utils/tarball.rs delete mode 100644 src/bootstrap/suggest.rs delete mode 100644 src/bootstrap/synthetic_targets.rs delete mode 100644 src/bootstrap/tarball.rs delete mode 100644 src/bootstrap/test.rs delete mode 100644 src/bootstrap/tool.rs delete mode 100644 src/bootstrap/toolstate.rs delete mode 100644 src/bootstrap/util.rs (limited to 'src/bootstrap') diff --git a/src/bootstrap/CHANGELOG.md b/src/bootstrap/CHANGELOG.md deleted file mode 100644 index 1aba07138..000000000 --- a/src/bootstrap/CHANGELOG.md +++ /dev/null @@ -1,71 +0,0 @@ -# Changelog - -All notable changes to bootstrap will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - - -## [Changes since the last major version] - -- Vendoring is no longer done automatically when building from git sources. To use vendoring, run `cargo vendor` manually, or use the pre-vendored `rustc-src` tarball. -- `llvm-libunwind` now accepts `in-tree` (formerly true), `system` or `no` (formerly false) [#77703](https://github.com/rust-lang/rust/pull/77703) -- The options `infodir`, `localstatedir`, and `gpg-password-file` are no longer allowed in config.toml. Previously, they were ignored without warning. Note that `infodir` and `localstatedir` are still accepted by `./configure`, with a warning. [#82451](https://github.com/rust-lang/rust/pull/82451) -- Change the names for `dist` commands to match the component they generate. [#90684](https://github.com/rust-lang/rust/pull/90684) -- The `build.fast-submodules` option has been removed. Fast submodule checkouts are enabled unconditionally. Automatic submodule handling can still be disabled with `build.submodules = false`. -- Several unsupported `./configure` options have been removed: `optimize`, `parallel-compiler`. These can still be enabled with `--set`, although it isn't recommended. -- `remote-test-server`'s `verbose` argument has been removed in favor of the `--verbose` flag -- `remote-test-server`'s `remote` argument has been removed in favor of the `--bind` flag. Use `--bind 0.0.0.0:12345` to replicate the behavior of the `remote` argument. -- `x.py fmt` now formats only files modified between the merge-base of HEAD and the last commit in the master branch of the rust-lang repository and the current working directory. To restore old behaviour, use `x.py fmt .`. The check mode is not affected by this change. [#105702](https://github.com/rust-lang/rust/pull/105702) -- The `llvm.version-check` config option has been removed. Older versions were never supported. If you still need to support older versions (e.g. you are applying custom patches), patch `check_llvm_version` in bootstrap to change the minimum version. [#108619](https://github.com/rust-lang/rust/pull/108619) -- The `rust.ignore-git` option has been renamed to `rust.omit-git-hash`. [#110059](https://github.com/rust-lang/rust/pull/110059) -- `--exclude` no longer accepts a `Kind` as part of a Step; instead it uses the top-level Kind of the subcommand. If this matches how you were already using --exclude (e.g. `x test --exclude test::std`), simply remove the kind: `--exclude std`. If you were using a kind that did not match the top-level subcommand, please open an issue explaining why you wanted this feature. - -### Non-breaking changes - -- `x.py check` needs opt-in to check tests (--all-targets) [#77473](https://github.com/rust-lang/rust/pull/77473) -- The default bootstrap profiles are now located at `bootstrap/defaults/config.$PROFILE.toml` (previously they were located at `bootstrap/defaults/config.toml.$PROFILE`) [#77558](https://github.com/rust-lang/rust/pull/77558) -- If you have Rust already installed, `x.py` will now infer the host target - from the default rust toolchain. [#78513](https://github.com/rust-lang/rust/pull/78513) -- Add options for enabling overflow checks, one for std (`overflow-checks-std`) and one for everything else (`overflow-checks`). Both default to false. -- Add llvm option `enable-warnings` to have control on llvm compilation warnings. Default to false. -- Add `rpath` option in `target` section to support set rpath option for each target independently. [#111242](https://github.com/rust-lang/rust/pull/111242) - - -## [Version 2] - 2020-09-25 - -- `host` now defaults to the value of `build` in all cases - + Previously `host` defaulted to an empty list when `target` was overridden, and to `build` otherwise - -### Non-breaking changes - -- Add `x.py setup` [#76631](https://github.com/rust-lang/rust/pull/76631) -- Add a changelog for x.py [#76626](https://github.com/rust-lang/rust/pull/76626) -- Optionally, download LLVM from CI on Linux and NixOS. This can be enabled with `download-ci-llvm = true` under `[llvm]`. - + [#76439](https://github.com/rust-lang/rust/pull/76349) - + [#76667](https://github.com/rust-lang/rust/pull/76667) - + [#76708](https://github.com/rust-lang/rust/pull/76708) -- Distribute rustc sources as part of `rustc-dev` [#76856](https://github.com/rust-lang/rust/pull/76856) -- Make the default stage for x.py configurable [#76625](https://github.com/rust-lang/rust/pull/76625). This can be enabled with `build-stage = N`, `doc-stage = N`, etc. -- Add a dedicated debug-logging option [#76588](https://github.com/rust-lang/rust/pull/76588). Previously, `debug-logging` could only be set with `debug-assertions`, slowing down the compiler more than necessary. -- Add sample defaults for x.py [#76628](https://github.com/rust-lang/rust/pull/76628) -- Add `--keep-stage-std`, which behaves like `keep-stage` but allows the stage - 0 compiler artifacts (i.e., stage1/bin/rustc) to be rebuilt if changed - [#77120](https://github.com/rust-lang/rust/pull/77120). -- File locking is now used to avoid collisions between multiple running instances of `x.py` (e.g. when using `rust-analyzer` and `x.py` at the same time). Note that Solaris and possibly other non Unix and non Windows systems don't support it [#108607](https://github.com/rust-lang/rust/pull/108607). This might possibly lead to build data corruption. - - -## [Version 1] - 2020-09-11 - -This is the first changelog entry, and it does not attempt to be an exhaustive list of features in x.py. -Instead, this documents the changes to bootstrap in the past 2 months. - -- Improve defaults in `x.py` [#73964](https://github.com/rust-lang/rust/pull/73964) - (see [blog post] for details) -- Set `ninja = true` by default [#74922](https://github.com/rust-lang/rust/pull/74922) -- Avoid trying to inversely cross-compile for build triple from host triples [#76415](https://github.com/rust-lang/rust/pull/76415) -- Allow blessing expect-tests in tools [#75975](https://github.com/rust-lang/rust/pull/75975) -- `x.py check` checks tests/examples/benches [#76258](https://github.com/rust-lang/rust/pull/76258) -- Fix `rust.use-lld` when linker is not set [#76326](https://github.com/rust-lang/rust/pull/76326) -- Build tests with LLD if `use-lld = true` was passed [#76378](https://github.com/rust-lang/rust/pull/76378) - -[blog post]: https://blog.rust-lang.org/inside-rust/2020/08/30/changes-to-x-py-defaults.html diff --git a/src/bootstrap/Cargo.lock b/src/bootstrap/Cargo.lock index f5220e361..57113b0ec 100644 --- a/src/bootstrap/Cargo.lock +++ b/src/bootstrap/Cargo.lock @@ -29,6 +29,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" + [[package]] name = "block-buffer" version = "0.10.2" @@ -50,6 +56,7 @@ dependencies = [ "fd-lock", "filetime", "hex", + "home", "ignore", "junction", "libc", @@ -104,40 +111,38 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.2.4" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "956ac1f6381d8d82ab4684768f89c0ea3afe66925ceadb4eeb3fc452ffc55d62" +checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b" dependencies = [ "clap_builder", "clap_derive", - "once_cell", ] [[package]] name = "clap_builder" -version = "4.2.4" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84080e799e54cff944f4b4a4b0e71630b0e0443b25b985175c7dddc1a859b749" +checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663" dependencies = [ "anstyle", - "bitflags", "clap_lex", ] [[package]] name = "clap_complete" -version = "4.2.2" +version = "4.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36774babb166352bb4f7b9cb16f781ffa3439d2a8f12cd31bea85a38c888fea3" +checksum = "e3ae8ba90b9d8b007efe66e55e48fb936272f5ca00349b5b0e89877520d35ea7" dependencies = [ "clap", ] [[package]] name = "clap_derive" -version = "4.2.0" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck", "proc-macro2", @@ -147,9 +152,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.4.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" [[package]] name = "cmake" @@ -175,16 +180,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crossbeam-channel" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - [[package]] name = "crossbeam-deque" version = "0.8.2" @@ -252,30 +247,19 @@ checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" [[package]] name = "errno" -version = "0.3.0" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0" +checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" dependencies = [ - "errno-dragonfly", "libc", "windows-sys", ] -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", - "libc", -] - [[package]] name = "fd-lock" -version = "3.0.11" +version = "3.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9799aefb4a2e4a01cc47610b1dd47c18ab13d991f27bbcaed9296f5a53d5cbad" +checksum = "ef033ed5e9bad94e55838ca0ca906db0e043f517adda0c8b79c7a8c66c93c1b5" dependencies = [ "cfg-if", "rustix", @@ -329,27 +313,21 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" - [[package]] name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "home" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "747309b4b440c06d57b0b25f2aee03ee9b5e5397d288c60e21fc709bb98a7408" +dependencies = [ + "winapi", +] + [[package]] name = "ignore" version = "0.4.18" @@ -368,17 +346,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "io-lifetimes" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" -dependencies = [ - "hermit-abi 0.3.2", - "libc", - "windows-sys", -] - [[package]] name = "itoa" version = "1.0.2" @@ -403,15 +370,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.140" +version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" [[package]] name = "linux-raw-sys" -version = "0.3.2" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f508063cc7bb32987c71511216bd5a32be15bccb6a80b52df8b9d7f01fc3aa2" +checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" [[package]] name = "log" @@ -457,16 +424,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "num_cpus" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" -dependencies = [ - "hermit-abi 0.1.19", - "libc", -] - [[package]] name = "object" version = "0.32.0" @@ -528,25 +485,22 @@ dependencies = [ [[package]] name = "rayon" -version = "1.6.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e060280438193c554f654141c9ea9417886713b7acd75974c85b18a69a88e0b" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" dependencies = [ - "crossbeam-deque", "either", "rayon-core", ] [[package]] name = "rayon-core" -version = "1.10.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", ] [[package]] @@ -555,7 +509,7 @@ version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -583,13 +537,12 @@ checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64" [[package]] name = "rustix" -version = "0.37.6" +version = "0.38.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d097081ed288dfe45699b72f5b5d648e5f15d64d900c7080273baa20c16a6849" +checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed" dependencies = [ - "bitflags", + "bitflags 2.4.1", "errno", - "io-lifetimes", "libc", "linux-raw-sys", "windows-sys", @@ -787,27 +740,37 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows" -version = "0.46.0" +version = "0.51.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca229916c5ee38c2f2bc1e9d8f04df975b4bd93f9955dc69fabb5d91270045c9" +dependencies = [ + "windows-core", + "windows-targets", +] + +[[package]] +name = "windows-core" +version = "0.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" dependencies = [ "windows-targets", ] [[package]] name = "windows-sys" -version = "0.45.0" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ "windows-targets", ] [[package]] name = "windows-targets" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", @@ -820,45 +783,45 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "xattr" diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index 9bf26948a..e4d359141 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -5,38 +5,48 @@ edition = "2021" build = "build.rs" default-run = "bootstrap" +[features] +build-metrics = ["sysinfo"] + [lib] -path = "lib.rs" +path = "src/lib.rs" doctest = false [[bin]] name = "bootstrap" -path = "bin/main.rs" +path = "src/bin/main.rs" test = false [[bin]] name = "rustc" -path = "bin/rustc.rs" +path = "src/bin/rustc.rs" test = false [[bin]] name = "rustdoc" -path = "bin/rustdoc.rs" +path = "src/bin/rustdoc.rs" test = false [[bin]] name = "sccache-plus-cl" -path = "bin/sccache-plus-cl.rs" +path = "src/bin/sccache-plus-cl.rs" test = false [dependencies] build_helper = { path = "../tools/build_helper" } +cc = "1.0.69" +clap = { version = "4.4.7", default-features = false, features = ["std", "usage", "help", "derive", "error-context"] } +clap_complete = "4.4.3" cmake = "0.1.38" filetime = "0.2" -cc = "1.0.69" -libc = "0.2" hex = "0.4" +home = "0.5.4" +ignore = "0.4.10" +libc = "0.2" object = { version = "0.32.0", default-features = false, features = ["archive", "coff", "read_core", "unaligned"] } +once_cell = "1.7.2" +opener = "0.5" +semver = "1.0.17" serde = "1.0.137" # Directly use serde_derive rather than through the derive feature of serde to allow building both # in parallel and to allow serde_json and toml to start building as soon as serde has been built. @@ -46,27 +56,21 @@ sha2 = "0.10" tar = "0.4" termcolor = "1.2.0" toml = "0.5" -ignore = "0.4.10" -opener = "0.5" -once_cell = "1.7.2" -xz2 = "0.1" walkdir = "2" +xz2 = "0.1" # Dependencies needed by the build-metrics feature sysinfo = { version = "0.26.0", optional = true } -clap = { version = "4.2.4", default-features = false, features = ["std", "usage", "help", "derive", "error-context"] } -clap_complete = "4.2.2" -semver = "1.0.17" # Solaris doesn't support flock() and thus fd-lock is not option now [target.'cfg(not(target_os = "solaris"))'.dependencies] -fd-lock = "3.0.8" +fd-lock = "3.0.13" [target.'cfg(windows)'.dependencies.junction] version = "1.0.0" [target.'cfg(windows)'.dependencies.windows] -version = "0.46.0" +version = "0.51.1" features = [ "Win32_Foundation", "Win32_Security", @@ -80,9 +84,6 @@ features = [ [dev-dependencies] pretty_assertions = "1.4" -[features] -build-metrics = ["sysinfo"] - # We care a lot about bootstrap's compile times, so don't include debuginfo for # dependencies, only bootstrap itself. [profile.dev] diff --git a/src/bootstrap/README.md b/src/bootstrap/README.md index 548281ca5..e7998a40a 100644 --- a/src/bootstrap/README.md +++ b/src/bootstrap/README.md @@ -181,11 +181,10 @@ Some general areas that you may be interested in modifying are: `Config` struct. * Adding a sanity check? Take a look at `bootstrap/sanity.rs`. -If you make a major change, please remember to: +If you make a major change on bootstrap configuration, please remember to: -+ Update `VERSION` in `src/bootstrap/main.rs`. -* Update `changelog-seen = N` in `config.example.toml`. -* Add an entry in `src/bootstrap/CHANGELOG.md`. ++ Update `CONFIG_CHANGE_HISTORY` in `src/bootstrap/lib.rs`. +* Update `change-id = {pull-request-id}` in `config.example.toml`. A 'major change' includes @@ -193,7 +192,7 @@ A 'major change' includes * A change in the default options. Changes that do not affect contributors to the compiler or users -building rustc from source don't need an update to `VERSION`. +building rustc from source don't need an update to `CONFIG_CHANGE_HISTORY`. If you have any questions, feel free to reach out on the `#t-infra/bootstrap` channel at [Rust Bootstrap Zulip server][rust-bootstrap-zulip]. When you encounter bugs, @@ -201,3 +200,8 @@ please file issues on the [Rust issue tracker][rust-issue-tracker]. [rust-bootstrap-zulip]: https://rust-lang.zulipchat.com/#narrow/stream/t-infra.2Fbootstrap [rust-issue-tracker]: https://github.com/rust-lang/rust/issues + +## Changelog + +Because we do not release bootstrap with versions, we also do not maintain CHANGELOG files. To +review the changes made to bootstrap, simply run `git log --no-merges --oneline -- src/bootstrap`. diff --git a/src/bootstrap/bin/_helper.rs b/src/bootstrap/bin/_helper.rs deleted file mode 100644 index 09aa471db..000000000 --- a/src/bootstrap/bin/_helper.rs +++ /dev/null @@ -1,24 +0,0 @@ -/// Parses the value of the "RUSTC_VERBOSE" environment variable and returns it as a `usize`. -/// If it was not defined, returns 0 by default. -/// -/// Panics if "RUSTC_VERBOSE" is defined with the value that is not an unsigned integer. -fn parse_rustc_verbose() -> usize { - use std::str::FromStr; - - match std::env::var("RUSTC_VERBOSE") { - Ok(s) => usize::from_str(&s).expect("RUSTC_VERBOSE should be an integer"), - Err(_) => 0, - } -} - -/// Parses the value of the "RUSTC_STAGE" environment variable and returns it as a `String`. -/// -/// If "RUSTC_STAGE" was not set, the program will be terminated with 101. -fn parse_rustc_stage() -> String { - std::env::var("RUSTC_STAGE").unwrap_or_else(|_| { - // Don't panic here; it's reasonable to try and run these shims directly. Give a helpful error instead. - eprintln!("rustc shim: fatal: RUSTC_STAGE was not set"); - eprintln!("rustc shim: note: use `x.py build -vvv` to see all environment variables set by bootstrap"); - exit(101); - }) -} diff --git a/src/bootstrap/bin/main.rs b/src/bootstrap/bin/main.rs deleted file mode 100644 index c497cabbd..000000000 --- a/src/bootstrap/bin/main.rs +++ /dev/null @@ -1,124 +0,0 @@ -//! rustbuild, the Rust build system -//! -//! This is the entry point for the build system used to compile the `rustc` -//! compiler. Lots of documentation can be found in the `README.md` file in the -//! parent directory, and otherwise documentation can be found throughout the `build` -//! directory in each respective module. - -#[cfg(all(any(unix, windows), not(target_os = "solaris")))] -use std::io::Write; -#[cfg(all(any(unix, windows), not(target_os = "solaris")))] -use std::process; -use std::{env, fs}; - -#[cfg(all(any(unix, windows), not(target_os = "solaris")))] -use bootstrap::t; -use bootstrap::{Build, Config, Subcommand, VERSION}; - -fn main() { - let args = env::args().skip(1).collect::>(); - let config = Config::parse(&args); - - #[cfg(all(any(unix, windows), not(target_os = "solaris")))] - let mut build_lock; - #[cfg(all(any(unix, windows), not(target_os = "solaris")))] - let _build_lock_guard; - #[cfg(all(any(unix, windows), not(target_os = "solaris")))] - // Display PID of process holding the lock - // PID will be stored in a lock file - { - let path = config.out.join("lock"); - let pid = match fs::read_to_string(&path) { - Ok(contents) => contents, - Err(_) => String::new(), - }; - - build_lock = - fd_lock::RwLock::new(t!(fs::OpenOptions::new().write(true).create(true).open(&path))); - _build_lock_guard = match build_lock.try_write() { - Ok(mut lock) => { - t!(lock.write(&process::id().to_string().as_ref())); - lock - } - err => { - drop(err); - println!("warning: build directory locked by process {pid}, waiting for lock"); - let mut lock = t!(build_lock.write()); - t!(lock.write(&process::id().to_string().as_ref())); - lock - } - }; - } - - #[cfg(any(not(any(unix, windows)), target_os = "solaris"))] - println!("warning: file locking not supported for target, not locking build directory"); - - // check_version warnings are not printed during setup - let changelog_suggestion = - if matches!(config.cmd, Subcommand::Setup { .. }) { None } else { check_version(&config) }; - - // NOTE: Since `./configure` generates a `config.toml`, distro maintainers will see the - // changelog warning, not the `x.py setup` message. - let suggest_setup = config.config.is_none() && !matches!(config.cmd, Subcommand::Setup { .. }); - if suggest_setup { - println!("warning: you have not made a `config.toml`"); - println!( - "help: consider running `./x.py setup` or copying `config.example.toml` by running \ - `cp config.example.toml config.toml`" - ); - } else if let Some(suggestion) = &changelog_suggestion { - println!("{suggestion}"); - } - - let pre_commit = config.src.join(".git").join("hooks").join("pre-commit"); - Build::new(config).build(); - - if suggest_setup { - println!("warning: you have not made a `config.toml`"); - println!( - "help: consider running `./x.py setup` or copying `config.example.toml` by running \ - `cp config.example.toml config.toml`" - ); - } else if let Some(suggestion) = &changelog_suggestion { - println!("{suggestion}"); - } - - // Give a warning if the pre-commit script is in pre-commit and not pre-push. - // HACK: Since the commit script uses hard links, we can't actually tell if it was installed by x.py setup or not. - // We could see if it's identical to src/etc/pre-push.sh, but pre-push may have been modified in the meantime. - // Instead, look for this comment, which is almost certainly not in any custom hook. - if fs::read_to_string(pre_commit).map_or(false, |contents| { - contents.contains("https://github.com/rust-lang/rust/issues/77620#issuecomment-705144570") - }) { - println!( - "warning: You have the pre-push script installed to .git/hooks/pre-commit. \ - Consider moving it to .git/hooks/pre-push instead, which runs less often." - ); - } - - if suggest_setup || changelog_suggestion.is_some() { - println!("note: this message was printed twice to make it more likely to be seen"); - } -} - -fn check_version(config: &Config) -> Option { - let mut msg = String::new(); - - let suggestion = if let Some(seen) = config.changelog_seen { - if seen != VERSION { - msg.push_str("warning: there have been changes to x.py since you last updated.\n"); - format!("update `config.toml` to use `changelog-seen = {VERSION}` instead") - } else { - return None; - } - } else { - msg.push_str("warning: x.py has made several changes recently you may want to look at\n"); - format!("add `changelog-seen = {VERSION}` at the top of `config.toml`") - }; - - msg.push_str("help: consider looking at the changes in `src/bootstrap/CHANGELOG.md`\n"); - msg.push_str("note: to silence this warning, "); - msg.push_str(&suggestion); - - Some(msg) -} diff --git a/src/bootstrap/bin/rustc.rs b/src/bootstrap/bin/rustc.rs deleted file mode 100644 index 20cd63b96..000000000 --- a/src/bootstrap/bin/rustc.rs +++ /dev/null @@ -1,410 +0,0 @@ -//! Shim which is passed to Cargo as "rustc" when running the bootstrap. -//! -//! This shim will take care of some various tasks that our build process -//! requires that Cargo can't quite do through normal configuration: -//! -//! 1. When compiling build scripts and build dependencies, we need a guaranteed -//! full standard library available. The only compiler which actually has -//! this is the snapshot, so we detect this situation and always compile with -//! the snapshot compiler. -//! 2. We pass a bunch of `--cfg` and other flags based on what we're compiling -//! (and this slightly differs based on a whether we're using a snapshot or -//! not), so we do that all here. -//! -//! This may one day be replaced by RUSTFLAGS, but the dynamic nature of -//! switching compilers for the bootstrap and for build scripts will probably -//! never get replaced. - -include!("../dylib_util.rs"); -include!("./_helper.rs"); - -use std::env; -use std::path::PathBuf; -use std::process::{exit, Child, Command}; -use std::time::Instant; - -fn main() { - let args = env::args_os().skip(1).collect::>(); - let arg = |name| args.windows(2).find(|args| args[0] == name).and_then(|args| args[1].to_str()); - - let stage = parse_rustc_stage(); - let verbose = parse_rustc_verbose(); - - // Detect whether or not we're a build script depending on whether --target - // is passed (a bit janky...) - let target = arg("--target"); - let version = args.iter().find(|w| &**w == "-vV"); - - // Use a different compiler for build scripts, since there may not yet be a - // libstd for the real compiler to use. However, if Cargo is attempting to - // determine the version of the compiler, the real compiler needs to be - // used. Currently, these two states are differentiated based on whether - // --target and -vV is/isn't passed. - let (rustc, libdir) = if target.is_none() && version.is_none() { - ("RUSTC_SNAPSHOT", "RUSTC_SNAPSHOT_LIBDIR") - } else { - ("RUSTC_REAL", "RUSTC_LIBDIR") - }; - - let sysroot = env::var_os("RUSTC_SYSROOT").expect("RUSTC_SYSROOT was not set"); - let on_fail = env::var_os("RUSTC_ON_FAIL").map(Command::new); - - let rustc = env::var_os(rustc).unwrap_or_else(|| panic!("{:?} was not set", rustc)); - let libdir = env::var_os(libdir).unwrap_or_else(|| panic!("{:?} was not set", libdir)); - let mut dylib_path = dylib_path(); - dylib_path.insert(0, PathBuf::from(&libdir)); - - let mut cmd = Command::new(rustc); - cmd.args(&args).env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); - - // Get the name of the crate we're compiling, if any. - let crate_name = arg("--crate-name"); - - if let Some(crate_name) = crate_name { - if let Some(target) = env::var_os("RUSTC_TIME") { - if target == "all" - || target.into_string().unwrap().split(',').any(|c| c.trim() == crate_name) - { - cmd.arg("-Ztime-passes"); - } - } - } - - // Print backtrace in case of ICE - if env::var("RUSTC_BACKTRACE_ON_ICE").is_ok() && env::var("RUST_BACKTRACE").is_err() { - cmd.env("RUST_BACKTRACE", "1"); - } - - if let Ok(lint_flags) = env::var("RUSTC_LINT_FLAGS") { - cmd.args(lint_flags.split_whitespace()); - } - - if target.is_some() { - // The stage0 compiler has a special sysroot distinct from what we - // actually downloaded, so we just always pass the `--sysroot` option, - // unless one is already set. - if !args.iter().any(|arg| arg == "--sysroot") { - cmd.arg("--sysroot").arg(&sysroot); - } - - // If we're compiling specifically the `panic_abort` crate then we pass - // the `-C panic=abort` option. Note that we do not do this for any - // other crate intentionally as this is the only crate for now that we - // ship with panic=abort. - // - // This... is a bit of a hack how we detect this. Ideally this - // information should be encoded in the crate I guess? Would likely - // require an RFC amendment to RFC 1513, however. - if crate_name == Some("panic_abort") { - cmd.arg("-C").arg("panic=abort"); - } - - // `-Ztls-model=initial-exec` must not be applied to proc-macros, see - // issue https://github.com/rust-lang/rust/issues/100530 - if env::var("RUSTC_TLS_MODEL_INITIAL_EXEC").is_ok() - && arg("--crate-type") != Some("proc-macro") - && !matches!(crate_name, Some("proc_macro2" | "quote" | "syn" | "synstructure")) - { - cmd.arg("-Ztls-model=initial-exec"); - } - } else { - // FIXME(rust-lang/cargo#5754) we shouldn't be using special env vars - // here, but rather Cargo should know what flags to pass rustc itself. - - // Override linker if necessary. - if let Ok(host_linker) = env::var("RUSTC_HOST_LINKER") { - cmd.arg(format!("-Clinker={host_linker}")); - } - if env::var_os("RUSTC_HOST_FUSE_LD_LLD").is_some() { - cmd.arg("-Clink-args=-fuse-ld=lld"); - } - - if let Ok(s) = env::var("RUSTC_HOST_CRT_STATIC") { - if s == "true" { - cmd.arg("-C").arg("target-feature=+crt-static"); - } - if s == "false" { - cmd.arg("-C").arg("target-feature=-crt-static"); - } - } - - // Cargo doesn't pass RUSTFLAGS to proc_macros: - // https://github.com/rust-lang/cargo/issues/4423 - // Thus, if we are on stage 0, we explicitly set `--cfg=bootstrap`. - // We also declare that the flag is expected, which we need to do to not - // get warnings about it being unexpected. - if stage == "0" { - cmd.arg("--cfg=bootstrap"); - } - cmd.arg("-Zunstable-options"); - cmd.arg("--check-cfg=values(bootstrap)"); - } - - if let Ok(map) = env::var("RUSTC_DEBUGINFO_MAP") { - cmd.arg("--remap-path-prefix").arg(&map); - } - - // Force all crates compiled by this compiler to (a) be unstable and (b) - // allow the `rustc_private` feature to link to other unstable crates - // also in the sysroot. We also do this for host crates, since those - // may be proc macros, in which case we might ship them. - if env::var_os("RUSTC_FORCE_UNSTABLE").is_some() { - cmd.arg("-Z").arg("force-unstable-if-unmarked"); - } - - // allow-features is handled from within this rustc wrapper because of - // issues with build scripts. Some packages use build scripts to - // dynamically detect if certain nightly features are available. - // There are different ways this causes problems: - // - // * rustix runs `rustc` on a small test program to see if the feature is - // available (and sets a `cfg` if it is). It does not honor - // CARGO_ENCODED_RUSTFLAGS. - // * proc-macro2 detects if `rustc -vV` says "nighty" or "dev" and enables - // nightly features. It will scan CARGO_ENCODED_RUSTFLAGS for - // -Zallow-features. Unfortunately CARGO_ENCODED_RUSTFLAGS is not set - // for build-dependencies when --target is used. - // - // The issues above means we can't just use RUSTFLAGS, and we can't use - // `cargo -Zallow-features=…`. Passing it through here ensures that it - // always gets set. Unfortunately that also means we need to enable more - // features than we really want (like those for proc-macro2), but there - // isn't much of a way around it. - // - // I think it is unfortunate that build scripts are doing this at all, - // since changes to nightly features can cause crates to break even if the - // user didn't want or care about the use of the nightly features. I think - // nightly features should be opt-in only. Unfortunately the dynamic - // checks are now too wide spread that we just need to deal with it. - // - // If you want to try to remove this, I suggest working with the crate - // authors to remove the dynamic checking. Another option is to pursue - // https://github.com/rust-lang/cargo/issues/11244 and - // https://github.com/rust-lang/cargo/issues/4423, which will likely be - // very difficult, but could help expose -Zallow-features into build - // scripts so they could try to honor them. - if let Ok(allow_features) = env::var("RUSTC_ALLOW_FEATURES") { - cmd.arg(format!("-Zallow-features={allow_features}")); - } - - if let Ok(flags) = env::var("MAGIC_EXTRA_RUSTFLAGS") { - for flag in flags.split(' ') { - cmd.arg(flag); - } - } - - let is_test = args.iter().any(|a| a == "--test"); - if verbose > 2 { - let rust_env_vars = - env::vars().filter(|(k, _)| k.starts_with("RUST") || k.starts_with("CARGO")); - let prefix = if is_test { "[RUSTC-SHIM] rustc --test" } else { "[RUSTC-SHIM] rustc" }; - let prefix = match crate_name { - Some(crate_name) => format!("{prefix} {crate_name}"), - None => prefix.to_string(), - }; - for (i, (k, v)) in rust_env_vars.enumerate() { - eprintln!("{prefix} env[{i}]: {k:?}={v:?}"); - } - eprintln!("{} working directory: {}", prefix, env::current_dir().unwrap().display()); - eprintln!( - "{} command: {:?}={:?} {:?}", - prefix, - dylib_path_var(), - env::join_paths(&dylib_path).unwrap(), - cmd, - ); - eprintln!("{prefix} sysroot: {sysroot:?}"); - eprintln!("{prefix} libdir: {libdir:?}"); - } - - let start = Instant::now(); - let (child, status) = { - let errmsg = format!("\nFailed to run:\n{cmd:?}\n-------------"); - let mut child = cmd.spawn().expect(&errmsg); - let status = child.wait().expect(&errmsg); - (child, status) - }; - - if env::var_os("RUSTC_PRINT_STEP_TIMINGS").is_some() - || env::var_os("RUSTC_PRINT_STEP_RUSAGE").is_some() - { - if let Some(crate_name) = crate_name { - let dur = start.elapsed(); - // If the user requested resource usage data, then - // include that in addition to the timing output. - let rusage_data = - env::var_os("RUSTC_PRINT_STEP_RUSAGE").and_then(|_| format_rusage_data(child)); - eprintln!( - "[RUSTC-TIMING] {} test:{} {}.{:03}{}{}", - crate_name, - is_test, - dur.as_secs(), - dur.subsec_millis(), - if rusage_data.is_some() { " " } else { "" }, - rusage_data.unwrap_or(String::new()), - ); - } - } - - if status.success() { - std::process::exit(0); - // note: everything below here is unreachable. do not put code that - // should run on success, after this block. - } - if verbose > 0 { - println!("\nDid not run successfully: {status}\n{cmd:?}\n-------------"); - } - - if let Some(mut on_fail) = on_fail { - on_fail.status().expect("Could not run the on_fail command"); - } - - // Preserve the exit code. In case of signal, exit with 0xfe since it's - // awkward to preserve this status in a cross-platform way. - match status.code() { - Some(i) => std::process::exit(i), - None => { - eprintln!("rustc exited with {status}"); - std::process::exit(0xfe); - } - } -} - -#[cfg(all(not(unix), not(windows)))] -// In the future we can add this for more platforms -fn format_rusage_data(_child: Child) -> Option { - None -} - -#[cfg(windows)] -fn format_rusage_data(child: Child) -> Option { - use std::os::windows::io::AsRawHandle; - - use windows::{ - Win32::Foundation::HANDLE, - Win32::System::ProcessStatus::{ - K32GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS, PROCESS_MEMORY_COUNTERS_EX, - }, - Win32::System::Threading::GetProcessTimes, - Win32::System::Time::FileTimeToSystemTime, - }; - - let handle = HANDLE(child.as_raw_handle() as isize); - - let mut user_filetime = Default::default(); - let mut user_time = Default::default(); - let mut kernel_filetime = Default::default(); - let mut kernel_time = Default::default(); - let mut memory_counters = PROCESS_MEMORY_COUNTERS::default(); - - unsafe { - GetProcessTimes( - handle, - &mut Default::default(), - &mut Default::default(), - &mut kernel_filetime, - &mut user_filetime, - ) - } - .ok() - .ok()?; - unsafe { FileTimeToSystemTime(&user_filetime, &mut user_time) }.ok().ok()?; - unsafe { FileTimeToSystemTime(&kernel_filetime, &mut kernel_time) }.ok().ok()?; - - // Unlike on Linux with RUSAGE_CHILDREN, this will only return memory information for the process - // with the given handle and none of that process's children. - unsafe { - K32GetProcessMemoryInfo( - handle, - &mut memory_counters, - std::mem::size_of::() as u32, - ) - } - .ok() - .ok()?; - - // Guide on interpreting these numbers: - // https://docs.microsoft.com/en-us/windows/win32/psapi/process-memory-usage-information - let peak_working_set = memory_counters.PeakWorkingSetSize / 1024; - let peak_page_file = memory_counters.PeakPagefileUsage / 1024; - let peak_paged_pool = memory_counters.QuotaPeakPagedPoolUsage / 1024; - let peak_nonpaged_pool = memory_counters.QuotaPeakNonPagedPoolUsage / 1024; - Some(format!( - "user: {USER_SEC}.{USER_USEC:03} \ - sys: {SYS_SEC}.{SYS_USEC:03} \ - peak working set (kb): {PEAK_WORKING_SET} \ - peak page file usage (kb): {PEAK_PAGE_FILE} \ - peak paged pool usage (kb): {PEAK_PAGED_POOL} \ - peak non-paged pool usage (kb): {PEAK_NONPAGED_POOL} \ - page faults: {PAGE_FAULTS}", - USER_SEC = user_time.wSecond + (user_time.wMinute * 60), - USER_USEC = user_time.wMilliseconds, - SYS_SEC = kernel_time.wSecond + (kernel_time.wMinute * 60), - SYS_USEC = kernel_time.wMilliseconds, - PEAK_WORKING_SET = peak_working_set, - PEAK_PAGE_FILE = peak_page_file, - PEAK_PAGED_POOL = peak_paged_pool, - PEAK_NONPAGED_POOL = peak_nonpaged_pool, - PAGE_FAULTS = memory_counters.PageFaultCount, - )) -} - -#[cfg(unix)] -/// Tries to build a string with human readable data for several of the rusage -/// fields. Note that we are focusing mainly on data that we believe to be -/// supplied on Linux (the `rusage` struct has other fields in it but they are -/// currently unsupported by Linux). -fn format_rusage_data(_child: Child) -> Option { - let rusage: libc::rusage = unsafe { - let mut recv = std::mem::zeroed(); - // -1 is RUSAGE_CHILDREN, which means to get the rusage for all children - // (and grandchildren, etc) processes that have respectively terminated - // and been waited for. - let retval = libc::getrusage(-1, &mut recv); - if retval != 0 { - return None; - } - recv - }; - // Mac OS X reports the maxrss in bytes, not kb. - let divisor = if env::consts::OS == "macos" { 1024 } else { 1 }; - let maxrss = (rusage.ru_maxrss + (divisor - 1)) / divisor; - - let mut init_str = format!( - "user: {USER_SEC}.{USER_USEC:03} \ - sys: {SYS_SEC}.{SYS_USEC:03} \ - max rss (kb): {MAXRSS}", - USER_SEC = rusage.ru_utime.tv_sec, - USER_USEC = rusage.ru_utime.tv_usec, - SYS_SEC = rusage.ru_stime.tv_sec, - SYS_USEC = rusage.ru_stime.tv_usec, - MAXRSS = maxrss - ); - - // The remaining rusage stats vary in platform support. So we treat - // uniformly zero values in each category as "not worth printing", since it - // either means no events of that type occurred, or that the platform - // does not support it. - - let minflt = rusage.ru_minflt; - let majflt = rusage.ru_majflt; - if minflt != 0 || majflt != 0 { - init_str.push_str(&format!(" page reclaims: {minflt} page faults: {majflt}")); - } - - let inblock = rusage.ru_inblock; - let oublock = rusage.ru_oublock; - if inblock != 0 || oublock != 0 { - init_str.push_str(&format!(" fs block inputs: {inblock} fs block outputs: {oublock}")); - } - - let nvcsw = rusage.ru_nvcsw; - let nivcsw = rusage.ru_nivcsw; - if nvcsw != 0 || nivcsw != 0 { - init_str.push_str(&format!( - " voluntary ctxt switches: {nvcsw} involuntary ctxt switches: {nivcsw}" - )); - } - - return Some(init_str); -} diff --git a/src/bootstrap/bin/rustdoc.rs b/src/bootstrap/bin/rustdoc.rs deleted file mode 100644 index 6561c1c19..000000000 --- a/src/bootstrap/bin/rustdoc.rs +++ /dev/null @@ -1,86 +0,0 @@ -//! Shim which is passed to Cargo as "rustdoc" when running the bootstrap. -//! -//! See comments in `src/bootstrap/rustc.rs` for more information. - -use std::env; -use std::ffi::OsString; -use std::path::PathBuf; -use std::process::{exit, Command}; - -include!("../dylib_util.rs"); - -include!("./_helper.rs"); - -fn main() { - let args = env::args_os().skip(1).collect::>(); - - let stage = parse_rustc_stage(); - let verbose = parse_rustc_verbose(); - - let rustdoc = env::var_os("RUSTDOC_REAL").expect("RUSTDOC_REAL was not set"); - let libdir = env::var_os("RUSTDOC_LIBDIR").expect("RUSTDOC_LIBDIR was not set"); - let sysroot = env::var_os("RUSTC_SYSROOT").expect("RUSTC_SYSROOT was not set"); - - // Detect whether or not we're a build script depending on whether --target - // is passed (a bit janky...) - let target = args.windows(2).find(|w| &*w[0] == "--target").and_then(|w| w[1].to_str()); - - let mut dylib_path = dylib_path(); - dylib_path.insert(0, PathBuf::from(libdir.clone())); - - let mut cmd = Command::new(rustdoc); - - if target.is_some() { - // The stage0 compiler has a special sysroot distinct from what we - // actually downloaded, so we just always pass the `--sysroot` option, - // unless one is already set. - if !args.iter().any(|arg| arg == "--sysroot") { - cmd.arg("--sysroot").arg(&sysroot); - } - } - - cmd.args(&args); - cmd.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); - - // Force all crates compiled by this compiler to (a) be unstable and (b) - // allow the `rustc_private` feature to link to other unstable crates - // also in the sysroot. - if env::var_os("RUSTC_FORCE_UNSTABLE").is_some() { - cmd.arg("-Z").arg("force-unstable-if-unmarked"); - } - if let Some(linker) = env::var_os("RUSTDOC_LINKER") { - let mut arg = OsString::from("-Clinker="); - arg.push(&linker); - cmd.arg(arg); - } - if let Ok(no_threads) = env::var("RUSTDOC_LLD_NO_THREADS") { - cmd.arg("-Clink-arg=-fuse-ld=lld"); - cmd.arg(format!("-Clink-arg=-Wl,{no_threads}")); - } - // Cargo doesn't pass RUSTDOCFLAGS to proc_macros: - // https://github.com/rust-lang/cargo/issues/4423 - // Thus, if we are on stage 0, we explicitly set `--cfg=bootstrap`. - // We also declare that the flag is expected, which we need to do to not - // get warnings about it being unexpected. - if stage == "0" { - cmd.arg("--cfg=bootstrap"); - } - cmd.arg("-Zunstable-options"); - cmd.arg("--check-cfg=values(bootstrap)"); - - if verbose > 1 { - eprintln!( - "rustdoc command: {:?}={:?} {:?}", - dylib_path_var(), - env::join_paths(&dylib_path).unwrap(), - cmd, - ); - eprintln!("sysroot: {sysroot:?}"); - eprintln!("libdir: {libdir:?}"); - } - - std::process::exit(match cmd.status() { - Ok(s) => s.code().unwrap_or(1), - Err(e) => panic!("\n\nfailed to run {cmd:?}: {e}\n\n"), - }) -} diff --git a/src/bootstrap/bin/sccache-plus-cl.rs b/src/bootstrap/bin/sccache-plus-cl.rs deleted file mode 100644 index 554c2dd4d..000000000 --- a/src/bootstrap/bin/sccache-plus-cl.rs +++ /dev/null @@ -1,38 +0,0 @@ -use std::env; -use std::process::{self, Command}; - -fn main() { - let target = env::var("SCCACHE_TARGET").unwrap(); - // Locate the actual compiler that we're invoking - env::set_var("CC", env::var_os("SCCACHE_CC").unwrap()); - env::set_var("CXX", env::var_os("SCCACHE_CXX").unwrap()); - let mut cfg = cc::Build::new(); - cfg.cargo_metadata(false) - .out_dir("/") - .target(&target) - .host(&target) - .opt_level(0) - .warnings(false) - .debug(false); - let compiler = cfg.get_compiler(); - - // Invoke sccache with said compiler - let sccache_path = env::var_os("SCCACHE_PATH").unwrap(); - let mut cmd = Command::new(&sccache_path); - cmd.arg(compiler.path()); - for &(ref k, ref v) in compiler.env() { - cmd.env(k, v); - } - for arg in env::args().skip(1) { - cmd.arg(arg); - } - - if let Ok(s) = env::var("SCCACHE_EXTRA_ARGS") { - for s in s.split_whitespace() { - cmd.arg(s); - } - } - - let status = cmd.status().expect("failed to spawn"); - process::exit(status.code().unwrap_or(2)) -} diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index fac0cdf20..5a84e37f8 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -211,7 +211,7 @@ def require(cmd, exit=True, exception=False): if exception: raise elif exit: - eprint("error: unable to run `{}`: {}".format(' '.join(cmd), exc)) + eprint("ERROR: unable to run `{}`: {}".format(' '.join(cmd), exc)) eprint("Please make sure it's installed and in the path.") sys.exit(1) return None @@ -681,7 +681,7 @@ class RustBuild(object): answer = self._should_fix_bins_and_dylibs = get_answer() if answer: - eprint("info: You seem to be using Nix.") + eprint("INFO: You seem to be using Nix.") return answer def fix_bin_or_dylib(self, fname): @@ -727,7 +727,7 @@ class RustBuild(object): "nix-build", "-E", nix_expr, "-o", nix_deps_dir, ]) except subprocess.CalledProcessError as reason: - eprint("warning: failed to call nix-build:", reason) + eprint("WARNING: failed to call nix-build:", reason) return self.nix_deps_dir = nix_deps_dir @@ -747,7 +747,7 @@ class RustBuild(object): try: subprocess.check_output([patchelf] + patchelf_args + [fname]) except subprocess.CalledProcessError as reason: - eprint("warning: failed to call patchelf:", reason) + eprint("WARNING: failed to call patchelf:", reason) return def rustc_stamp(self): @@ -954,6 +954,13 @@ class RustBuild(object): if deny_warnings: env["RUSTFLAGS"] += " -Dwarnings" + # Add RUSTFLAGS_BOOTSTRAP to RUSTFLAGS for bootstrap compilation. + # Note that RUSTFLAGS_BOOTSTRAP should always be added to the end of + # RUSTFLAGS to be actually effective (e.g., if we have `-Dwarnings` in + # RUSTFLAGS, passing `-Awarnings` from RUSTFLAGS_BOOTSTRAP should override it). + if "RUSTFLAGS_BOOTSTRAP" in env: + env["RUSTFLAGS"] += " " + env["RUSTFLAGS_BOOTSTRAP"] + env["PATH"] = os.path.join(self.bin_root(), "bin") + \ os.pathsep + env["PATH"] if not os.path.isfile(self.cargo()): @@ -998,7 +1005,7 @@ class RustBuild(object): if 'SUDO_USER' in os.environ and not self.use_vendored_sources: if os.getuid() == 0: self.use_vendored_sources = True - eprint('info: looks like you\'re trying to run this command as root') + eprint('INFO: looks like you\'re trying to run this command as root') eprint(' and so in order to preserve your $HOME this will now') eprint(' use vendored sources by default.') @@ -1010,14 +1017,14 @@ class RustBuild(object): "--sync ./src/tools/rust-analyzer/Cargo.toml " \ "--sync ./compiler/rustc_codegen_cranelift/Cargo.toml " \ "--sync ./src/bootstrap/Cargo.toml " - eprint('error: vendoring required, but vendor directory does not exist.') + eprint('ERROR: vendoring required, but vendor directory does not exist.') eprint(' Run `cargo vendor {}` to initialize the ' 'vendor directory.'.format(sync_dirs)) eprint('Alternatively, use the pre-vendored `rustc-src` dist component.') raise Exception("{} not found".format(vendor_dir)) if not os.path.exists(cargo_dir): - eprint('error: vendoring required, but .cargo/config does not exist.') + eprint('ERROR: vendoring required, but .cargo/config does not exist.') raise Exception("{} not found".format(cargo_dir)) else: if os.path.exists(cargo_dir): @@ -1042,6 +1049,12 @@ def bootstrap(args): """Configure, fetch, build and run the initial bootstrap""" rust_root = os.path.abspath(os.path.join(__file__, '../../..')) + if not os.path.exists(os.path.join(rust_root, '.git')) and \ + os.path.exists(os.path.join(rust_root, '.github')): + eprint("warn: Looks like you are trying to bootstrap Rust from a source that is neither a " + "git clone nor distributed tarball.\nThis build may fail due to missing submodules " + "unless you put them in place manually.") + # Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, # then `config.toml` in the root directory. toml_path = args.config or os.getenv('RUST_BOOTSTRAP_CONFIG') @@ -1112,7 +1125,7 @@ def main(): # process has to happen before anything is printed out. if help_triggered: eprint( - "info: Downloading and building bootstrap before processing --help command.\n" + "INFO: Downloading and building bootstrap before processing --help command.\n" " See src/bootstrap/README.md for help with common commands.") exit_code = 0 diff --git a/src/bootstrap/bootstrap_test.py b/src/bootstrap/bootstrap_test.py index dc06a4c97..6da410ed2 100644 --- a/src/bootstrap/bootstrap_test.py +++ b/src/bootstrap/bootstrap_test.py @@ -34,7 +34,7 @@ def serialize_and_parse(configure_args, bootstrap_args=None): # Verify this is actually valid TOML. tomllib.loads(build.config_toml) except ImportError: - print("warning: skipping TOML validation, need at least python 3.11", file=sys.stderr) + print("WARNING: skipping TOML validation, need at least python 3.11", file=sys.stderr) return build @@ -103,7 +103,6 @@ class GenerateAndParseConfig(unittest.TestCase): """Test that we can serialize and deserialize a config.toml file""" def test_no_args(self): build = serialize_and_parse([]) - self.assertEqual(build.get_toml("changelog-seen"), '2') self.assertEqual(build.get_toml("profile"), 'dist') self.assertIsNone(build.get_toml("llvm.download-ci-llvm")) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs deleted file mode 100644 index 46a62eed9..000000000 --- a/src/bootstrap/builder.rs +++ /dev/null @@ -1,2318 +0,0 @@ -use std::any::{type_name, Any}; -use std::cell::{Cell, RefCell}; -use std::collections::BTreeSet; -use std::env; -use std::ffi::OsStr; -use std::fmt::{Debug, Write}; -use std::fs::{self, File}; -use std::hash::Hash; -use std::io::{BufRead, BufReader}; -use std::ops::Deref; -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::time::{Duration, Instant}; - -use crate::cache::{Cache, Interned, INTERNER}; -use crate::config::{DryRun, SplitDebuginfo, TargetSelection}; -use crate::doc; -use crate::flags::{Color, Subcommand}; -use crate::install; -use crate::llvm; -use crate::run; -use crate::setup; -use crate::test; -use crate::tool::{self, SourceType}; -use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir, output, t}; -use crate::EXTRA_CHECK_CFGS; -use crate::{check, compile, Crate}; -use crate::{clean, dist}; -use crate::{Build, CLang, DocTests, GitRepo, Mode}; - -pub use crate::Compiler; -// FIXME: -// - use std::lazy for `Lazy` -// - use std::cell for `OnceCell` -// Once they get stabilized and reach beta. -use clap::ValueEnum; -use once_cell::sync::{Lazy, OnceCell}; - -pub struct Builder<'a> { - pub build: &'a Build, - pub top_stage: u32, - pub kind: Kind, - cache: Cache, - stack: RefCell>>, - time_spent_on_dependencies: Cell, - pub paths: Vec, -} - -impl<'a> Deref for Builder<'a> { - type Target = Build; - - fn deref(&self) -> &Self::Target { - self.build - } -} - -pub trait Step: 'static + Clone + Debug + PartialEq + Eq + Hash { - /// `PathBuf` when directories are created or to return a `Compiler` once - /// it's been assembled. - type Output: Clone; - - /// Whether this step is run by default as part of its respective phase. - /// `true` here can still be overwritten by `should_run` calling `default_condition`. - const DEFAULT: bool = false; - - /// If true, then this rule should be skipped if --target was specified, but --host was not - const ONLY_HOSTS: bool = false; - - /// Primary function to execute this rule. Can call `builder.ensure()` - /// with other steps to run those. - fn run(self, builder: &Builder<'_>) -> Self::Output; - - /// When bootstrap is passed a set of paths, this controls whether this rule - /// will execute. However, it does not get called in a "default" context - /// when we are not passed any paths; in that case, `make_run` is called - /// directly. - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_>; - - /// Builds up a "root" rule, either as a default rule or from a path passed - /// to us. - /// - /// When path is `None`, we are executing in a context where no paths were - /// passed. When `./x.py build` is run, for example, this rule could get - /// called if it is in the correct list below with a path of `None`. - fn make_run(_run: RunConfig<'_>) { - // It is reasonable to not have an implementation of make_run for rules - // who do not want to get called from the root context. This means that - // they are likely dependencies (e.g., sysroot creation) or similar, and - // as such calling them from ./x.py isn't logical. - unimplemented!() - } -} - -pub struct RunConfig<'a> { - pub builder: &'a Builder<'a>, - pub target: TargetSelection, - pub paths: Vec, -} - -impl RunConfig<'_> { - pub fn build_triple(&self) -> TargetSelection { - self.builder.build.build - } - - /// Return a list of crate names selected by `run.paths`. - #[track_caller] - pub fn cargo_crates_in_set(&self) -> Interned> { - let mut crates = Vec::new(); - for krate in &self.paths { - let path = krate.assert_single_path(); - let Some(crate_name) = self.builder.crate_paths.get(&path.path) else { - panic!("missing crate for path {}", path.path.display()) - }; - crates.push(crate_name.to_string()); - } - INTERNER.intern_list(crates) - } - - /// Given an `alias` selected by the `Step` and the paths passed on the command line, - /// return a list of the crates that should be built. - /// - /// Normally, people will pass *just* `library` if they pass it. - /// But it's possible (although strange) to pass something like `library std core`. - /// Build all crates anyway, as if they hadn't passed the other args. - pub fn make_run_crates(&self, alias: Alias) -> Interned> { - let has_alias = - self.paths.iter().any(|set| set.assert_single_path().path.ends_with(alias.as_str())); - if !has_alias { - return self.cargo_crates_in_set(); - } - - let crates = match alias { - Alias::Library => self.builder.in_tree_crates("sysroot", Some(self.target)), - Alias::Compiler => self.builder.in_tree_crates("rustc-main", Some(self.target)), - }; - - let crate_names = crates.into_iter().map(|krate| krate.name.to_string()).collect(); - INTERNER.intern_list(crate_names) - } -} - -#[derive(Debug, Copy, Clone)] -pub enum Alias { - Library, - Compiler, -} - -impl Alias { - fn as_str(self) -> &'static str { - match self { - Alias::Library => "library", - Alias::Compiler => "compiler", - } - } -} - -/// A description of the crates in this set, suitable for passing to `builder.info`. -/// -/// `crates` should be generated by [`RunConfig::cargo_crates_in_set`]. -pub fn crate_description(crates: &[impl AsRef]) -> String { - if crates.is_empty() { - return "".into(); - } - - let mut descr = String::from(" {"); - descr.push_str(crates[0].as_ref()); - for krate in &crates[1..] { - descr.push_str(", "); - descr.push_str(krate.as_ref()); - } - descr.push('}'); - descr -} - -struct StepDescription { - default: bool, - only_hosts: bool, - should_run: fn(ShouldRun<'_>) -> ShouldRun<'_>, - make_run: fn(RunConfig<'_>), - name: &'static str, - kind: Kind, -} - -#[derive(Clone, PartialOrd, Ord, PartialEq, Eq)] -pub struct TaskPath { - pub path: PathBuf, - pub kind: Option, -} - -impl Debug for TaskPath { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - if let Some(kind) = &self.kind { - write!(f, "{}::", kind.as_str())?; - } - write!(f, "{}", self.path.display()) - } -} - -/// Collection of paths used to match a task rule. -#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)] -pub enum PathSet { - /// A collection of individual paths or aliases. - /// - /// These are generally matched as a path suffix. For example, a - /// command-line value of `std` will match if `library/std` is in the - /// set. - /// - /// NOTE: the paths within a set should always be aliases of one another. - /// For example, `src/librustdoc` and `src/tools/rustdoc` should be in the same set, - /// but `library/core` and `library/std` generally should not, unless there's no way (for that Step) - /// to build them separately. - Set(BTreeSet), - /// A "suite" of paths. - /// - /// These can match as a path suffix (like `Set`), or as a prefix. For - /// example, a command-line value of `tests/ui/abi/variadic-ffi.rs` - /// will match `tests/ui`. A command-line value of `ui` would also - /// match `tests/ui`. - Suite(TaskPath), -} - -impl PathSet { - fn empty() -> PathSet { - PathSet::Set(BTreeSet::new()) - } - - fn one>(path: P, kind: Kind) -> PathSet { - let mut set = BTreeSet::new(); - set.insert(TaskPath { path: path.into(), kind: Some(kind) }); - PathSet::Set(set) - } - - fn has(&self, needle: &Path, module: Kind) -> bool { - match self { - PathSet::Set(set) => set.iter().any(|p| Self::check(p, needle, module)), - PathSet::Suite(suite) => Self::check(suite, needle, module), - } - } - - // internal use only - fn check(p: &TaskPath, needle: &Path, module: Kind) -> bool { - if let Some(p_kind) = &p.kind { - p.path.ends_with(needle) && *p_kind == module - } else { - p.path.ends_with(needle) - } - } - - /// Return all `TaskPath`s in `Self` that contain any of the `needles`, removing the - /// matched needles. - /// - /// This is used for `StepDescription::krate`, which passes all matching crates at once to - /// `Step::make_run`, rather than calling it many times with a single crate. - /// See `tests.rs` for examples. - fn intersection_removing_matches(&self, needles: &mut Vec<&Path>, module: Kind) -> PathSet { - let mut check = |p| { - for (i, n) in needles.iter().enumerate() { - let matched = Self::check(p, n, module); - if matched { - needles.remove(i); - return true; - } - } - false - }; - match self { - PathSet::Set(set) => PathSet::Set(set.iter().filter(|&p| check(p)).cloned().collect()), - PathSet::Suite(suite) => { - if check(suite) { - self.clone() - } else { - PathSet::empty() - } - } - } - } - - /// A convenience wrapper for Steps which know they have no aliases and all their sets contain only a single path. - /// - /// This can be used with [`ShouldRun::crate_or_deps`], [`ShouldRun::path`], or [`ShouldRun::alias`]. - #[track_caller] - pub fn assert_single_path(&self) -> &TaskPath { - match self { - PathSet::Set(set) => { - assert_eq!(set.len(), 1, "called assert_single_path on multiple paths"); - set.iter().next().unwrap() - } - PathSet::Suite(_) => unreachable!("called assert_single_path on a Suite path"), - } - } -} - -impl StepDescription { - fn from(kind: Kind) -> StepDescription { - StepDescription { - default: S::DEFAULT, - only_hosts: S::ONLY_HOSTS, - should_run: S::should_run, - make_run: S::make_run, - name: std::any::type_name::(), - kind, - } - } - - fn maybe_run(&self, builder: &Builder<'_>, mut pathsets: Vec) { - pathsets.retain(|set| !self.is_excluded(builder, set)); - - if pathsets.is_empty() { - return; - } - - // Determine the targets participating in this rule. - let targets = if self.only_hosts { &builder.hosts } else { &builder.targets }; - - for target in targets { - let run = RunConfig { builder, paths: pathsets.clone(), target: *target }; - (self.make_run)(run); - } - } - - fn is_excluded(&self, builder: &Builder<'_>, pathset: &PathSet) -> bool { - if builder.config.skip.iter().any(|e| pathset.has(&e, builder.kind)) { - if !matches!(builder.config.dry_run, DryRun::SelfCheck) { - println!("Skipping {pathset:?} because it is excluded"); - } - return true; - } - - if !builder.config.skip.is_empty() && !matches!(builder.config.dry_run, DryRun::SelfCheck) { - builder.verbose(&format!( - "{:?} not skipped for {:?} -- not in {:?}", - pathset, self.name, builder.config.skip - )); - } - false - } - - fn run(v: &[StepDescription], builder: &Builder<'_>, paths: &[PathBuf]) { - let should_runs = v - .iter() - .map(|desc| (desc.should_run)(ShouldRun::new(builder, desc.kind))) - .collect::>(); - - // sanity checks on rules - for (desc, should_run) in v.iter().zip(&should_runs) { - assert!( - !should_run.paths.is_empty(), - "{:?} should have at least one pathset", - desc.name - ); - } - - if paths.is_empty() || builder.config.include_default_paths { - for (desc, should_run) in v.iter().zip(&should_runs) { - if desc.default && should_run.is_really_default() { - desc.maybe_run(builder, should_run.paths.iter().cloned().collect()); - } - } - } - - // strip CurDir prefix if present - let mut paths: Vec<_> = - paths.into_iter().map(|p| p.strip_prefix(".").unwrap_or(p)).collect(); - - // Handle all test suite paths. - // (This is separate from the loop below to avoid having to handle multiple paths in `is_suite_path` somehow.) - paths.retain(|path| { - for (desc, should_run) in v.iter().zip(&should_runs) { - if let Some(suite) = should_run.is_suite_path(&path) { - desc.maybe_run(builder, vec![suite.clone()]); - return false; - } - } - true - }); - - if paths.is_empty() { - return; - } - - // Handle all PathSets. - for (desc, should_run) in v.iter().zip(&should_runs) { - let pathsets = should_run.pathset_for_paths_removing_matches(&mut paths, desc.kind); - if !pathsets.is_empty() { - desc.maybe_run(builder, pathsets); - } - } - - if !paths.is_empty() { - eprintln!("error: no `{}` rules matched {:?}", builder.kind.as_str(), paths,); - eprintln!( - "help: run `x.py {} --help --verbose` to show a list of available paths", - builder.kind.as_str() - ); - eprintln!( - "note: if you are adding a new Step to bootstrap itself, make sure you register it with `describe!`" - ); - crate::exit!(1); - } - } -} - -enum ReallyDefault<'a> { - Bool(bool), - Lazy(Lazy bool + 'a>>), -} - -pub struct ShouldRun<'a> { - pub builder: &'a Builder<'a>, - kind: Kind, - - // use a BTreeSet to maintain sort order - paths: BTreeSet, - - // If this is a default rule, this is an additional constraint placed on - // its run. Generally something like compiler docs being enabled. - is_really_default: ReallyDefault<'a>, -} - -impl<'a> ShouldRun<'a> { - fn new(builder: &'a Builder<'_>, kind: Kind) -> ShouldRun<'a> { - ShouldRun { - builder, - kind, - paths: BTreeSet::new(), - is_really_default: ReallyDefault::Bool(true), // by default no additional conditions - } - } - - pub fn default_condition(mut self, cond: bool) -> Self { - self.is_really_default = ReallyDefault::Bool(cond); - self - } - - pub fn lazy_default_condition(mut self, lazy_cond: Box bool + 'a>) -> Self { - self.is_really_default = ReallyDefault::Lazy(Lazy::new(lazy_cond)); - self - } - - pub fn is_really_default(&self) -> bool { - match &self.is_really_default { - ReallyDefault::Bool(val) => *val, - ReallyDefault::Lazy(lazy) => *lazy.deref(), - } - } - - /// Indicates it should run if the command-line selects the given crate or - /// any of its (local) dependencies. - /// - /// `make_run` will be called a single time with all matching command-line paths. - pub fn crate_or_deps(self, name: &str) -> Self { - let crates = self.builder.in_tree_crates(name, None); - self.crates(crates) - } - - /// Indicates it should run if the command-line selects any of the given crates. - /// - /// `make_run` will be called a single time with all matching command-line paths. - /// - /// Prefer [`ShouldRun::crate_or_deps`] to this function where possible. - pub(crate) fn crates(mut self, crates: Vec<&Crate>) -> Self { - for krate in crates { - let path = krate.local_path(self.builder); - self.paths.insert(PathSet::one(path, self.kind)); - } - self - } - - // single alias, which does not correspond to any on-disk path - pub fn alias(mut self, alias: &str) -> Self { - // exceptional case for `Kind::Setup` because its `library` - // and `compiler` options would otherwise naively match with - // `compiler` and `library` folders respectively. - assert!( - self.kind == Kind::Setup || !self.builder.src.join(alias).exists(), - "use `builder.path()` for real paths: {alias}" - ); - self.paths.insert(PathSet::Set( - std::iter::once(TaskPath { path: alias.into(), kind: Some(self.kind) }).collect(), - )); - self - } - - // single, non-aliased path - pub fn path(self, path: &str) -> Self { - self.paths(&[path]) - } - - /// Multiple aliases for the same job. - /// - /// This differs from [`path`] in that multiple calls to path will end up calling `make_run` - /// multiple times, whereas a single call to `paths` will only ever generate a single call to - /// `paths`. - /// - /// This is analogous to `all_krates`, although `all_krates` is gone now. Prefer [`path`] where possible. - /// - /// [`path`]: ShouldRun::path - pub fn paths(mut self, paths: &[&str]) -> Self { - static SUBMODULES_PATHS: OnceCell> = OnceCell::new(); - - let init_submodules_paths = |src: &PathBuf| { - let file = File::open(src.join(".gitmodules")).unwrap(); - - let mut submodules_paths = vec![]; - for line in BufReader::new(file).lines() { - if let Ok(line) = line { - let line = line.trim(); - - if line.starts_with("path") { - let actual_path = - line.split(' ').last().expect("Couldn't get value of path"); - submodules_paths.push(actual_path.to_owned()); - } - } - } - - submodules_paths - }; - - let submodules_paths = - SUBMODULES_PATHS.get_or_init(|| init_submodules_paths(&self.builder.src)); - - self.paths.insert(PathSet::Set( - paths - .iter() - .map(|p| { - // assert only if `p` isn't submodule - if submodules_paths.iter().find(|sm_p| p.contains(*sm_p)).is_none() { - assert!( - self.builder.src.join(p).exists(), - "`should_run.paths` should correspond to real on-disk paths - use `alias` if there is no relevant path: {}", - p - ); - } - - TaskPath { path: p.into(), kind: Some(self.kind) } - }) - .collect(), - )); - self - } - - /// Handles individual files (not directories) within a test suite. - fn is_suite_path(&self, requested_path: &Path) -> Option<&PathSet> { - self.paths.iter().find(|pathset| match pathset { - PathSet::Suite(suite) => requested_path.starts_with(&suite.path), - PathSet::Set(_) => false, - }) - } - - pub fn suite_path(mut self, suite: &str) -> Self { - self.paths.insert(PathSet::Suite(TaskPath { path: suite.into(), kind: Some(self.kind) })); - self - } - - // allows being more explicit about why should_run in Step returns the value passed to it - pub fn never(mut self) -> ShouldRun<'a> { - self.paths.insert(PathSet::empty()); - self - } - - /// Given a set of requested paths, return the subset which match the Step for this `ShouldRun`, - /// removing the matches from `paths`. - /// - /// NOTE: this returns multiple PathSets to allow for the possibility of multiple units of work - /// within the same step. For example, `test::Crate` allows testing multiple crates in the same - /// cargo invocation, which are put into separate sets because they aren't aliases. - /// - /// The reason we return PathSet instead of PathBuf is to allow for aliases that mean the same thing - /// (for now, just `all_krates` and `paths`, but we may want to add an `aliases` function in the future?) - fn pathset_for_paths_removing_matches( - &self, - paths: &mut Vec<&Path>, - kind: Kind, - ) -> Vec { - let mut sets = vec![]; - for pathset in &self.paths { - let subset = pathset.intersection_removing_matches(paths, kind); - if subset != PathSet::empty() { - sets.push(subset); - } - } - sets - } -} - -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] -pub enum Kind { - #[clap(alias = "b")] - Build, - #[clap(alias = "c")] - Check, - Clippy, - Fix, - Format, - #[clap(alias = "t")] - Test, - Bench, - #[clap(alias = "d")] - Doc, - Clean, - Dist, - Install, - #[clap(alias = "r")] - Run, - Setup, - Suggest, -} - -impl Kind { - pub fn parse(string: &str) -> Option { - // these strings, including the one-letter aliases, must match the x.py help text - Some(match string { - "build" | "b" => Kind::Build, - "check" | "c" => Kind::Check, - "clippy" => Kind::Clippy, - "fix" => Kind::Fix, - "fmt" => Kind::Format, - "test" | "t" => Kind::Test, - "bench" => Kind::Bench, - "doc" | "d" => Kind::Doc, - "clean" => Kind::Clean, - "dist" => Kind::Dist, - "install" => Kind::Install, - "run" | "r" => Kind::Run, - "setup" => Kind::Setup, - "suggest" => Kind::Suggest, - _ => return None, - }) - } - - pub fn as_str(&self) -> &'static str { - match self { - Kind::Build => "build", - Kind::Check => "check", - Kind::Clippy => "clippy", - Kind::Fix => "fix", - Kind::Format => "fmt", - Kind::Test => "test", - Kind::Bench => "bench", - Kind::Doc => "doc", - Kind::Clean => "clean", - Kind::Dist => "dist", - Kind::Install => "install", - Kind::Run => "run", - Kind::Setup => "setup", - Kind::Suggest => "suggest", - } - } - - pub fn description(&self) -> String { - match self { - Kind::Test => "Testing", - Kind::Bench => "Benchmarking", - Kind::Doc => "Documenting", - Kind::Run => "Running", - Kind::Suggest => "Suggesting", - _ => { - let title_letter = self.as_str()[0..1].to_ascii_uppercase(); - return format!("{title_letter}{}ing", &self.as_str()[1..]); - } - } - .to_owned() - } -} - -impl<'a> Builder<'a> { - fn get_step_descriptions(kind: Kind) -> Vec { - macro_rules! describe { - ($($rule:ty),+ $(,)?) => {{ - vec![$(StepDescription::from::<$rule>(kind)),+] - }}; - } - match kind { - Kind::Build => describe!( - compile::Std, - compile::Rustc, - compile::Assemble, - compile::CodegenBackend, - compile::StartupObjects, - tool::BuildManifest, - tool::Rustbook, - tool::ErrorIndex, - tool::UnstableBookGen, - tool::Tidy, - tool::Linkchecker, - tool::CargoTest, - tool::Compiletest, - tool::RemoteTestServer, - tool::RemoteTestClient, - tool::RustInstaller, - tool::Cargo, - tool::Rls, - tool::RustAnalyzer, - tool::RustAnalyzerProcMacroSrv, - tool::RustDemangler, - tool::Rustdoc, - tool::Clippy, - tool::CargoClippy, - llvm::Llvm, - llvm::Sanitizers, - tool::Rustfmt, - tool::Miri, - tool::CargoMiri, - llvm::Lld, - llvm::CrtBeginEnd, - tool::RustdocGUITest, - tool::OptimizedDist, - tool::CoverageDump, - ), - Kind::Check | Kind::Clippy | Kind::Fix => describe!( - check::Std, - check::Rustc, - check::Rustdoc, - check::CodegenBackend, - check::Clippy, - check::Miri, - check::CargoMiri, - check::MiroptTestTools, - check::Rls, - check::Rustfmt, - check::RustAnalyzer, - check::Bootstrap - ), - Kind::Test => describe!( - crate::toolstate::ToolStateCheck, - test::ExpandYamlAnchors, - test::Tidy, - test::Ui, - test::RunPassValgrind, - test::CoverageMap, - test::RunCoverage, - test::MirOpt, - test::Codegen, - test::CodegenUnits, - test::Assembly, - test::Incremental, - test::Debuginfo, - test::UiFullDeps, - test::CodegenCranelift, - test::Rustdoc, - test::RunCoverageRustdoc, - test::Pretty, - test::Crate, - test::CrateLibrustc, - test::CrateRustdoc, - test::CrateRustdocJsonTypes, - test::CrateBootstrap, - test::Linkcheck, - test::TierCheck, - test::Cargotest, - test::Cargo, - test::RustAnalyzer, - test::ErrorIndex, - test::Distcheck, - test::RunMakeFullDeps, - test::Nomicon, - test::Reference, - test::RustdocBook, - test::RustByExample, - test::TheBook, - test::UnstableBook, - test::RustcBook, - test::LintDocs, - test::RustcGuide, - test::EmbeddedBook, - test::EditionGuide, - test::Rustfmt, - test::Miri, - test::Clippy, - test::RustDemangler, - test::CompiletestTest, - test::RustdocJSStd, - test::RustdocJSNotStd, - test::RustdocGUI, - test::RustdocTheme, - test::RustdocUi, - test::RustdocJson, - test::HtmlCheck, - test::RustInstaller, - // Run bootstrap close to the end as it's unlikely to fail - test::Bootstrap, - // Run run-make last, since these won't pass without make on Windows - test::RunMake, - ), - Kind::Bench => describe!(test::Crate, test::CrateLibrustc), - Kind::Doc => describe!( - doc::UnstableBook, - doc::UnstableBookGen, - doc::TheBook, - doc::Standalone, - doc::Std, - doc::Rustc, - doc::Rustdoc, - doc::Rustfmt, - doc::ErrorIndex, - doc::Nomicon, - doc::Reference, - doc::RustdocBook, - doc::RustByExample, - doc::RustcBook, - doc::Cargo, - doc::CargoBook, - doc::Clippy, - doc::ClippyBook, - doc::Miri, - doc::EmbeddedBook, - doc::EditionGuide, - doc::StyleGuide, - doc::Tidy, - doc::Bootstrap, - ), - Kind::Dist => describe!( - dist::Docs, - dist::RustcDocs, - dist::JsonDocs, - dist::Mingw, - dist::Rustc, - dist::Std, - dist::RustcDev, - dist::Analysis, - dist::Src, - dist::Cargo, - dist::Rls, - dist::RustAnalyzer, - dist::Rustfmt, - dist::RustDemangler, - dist::Clippy, - dist::Miri, - dist::LlvmTools, - dist::RustDev, - dist::Bootstrap, - dist::Extended, - // It seems that PlainSourceTarball somehow changes how some of the tools - // perceive their dependencies (see #93033) which would invalidate fingerprints - // and force us to rebuild tools after vendoring dependencies. - // To work around this, create the Tarball after building all the tools. - dist::PlainSourceTarball, - dist::BuildManifest, - dist::ReproducibleArtifacts, - ), - Kind::Install => describe!( - install::Docs, - install::Std, - install::Cargo, - install::RustAnalyzer, - install::Rustfmt, - install::RustDemangler, - install::Clippy, - install::Miri, - install::LlvmTools, - install::Src, - install::Rustc - ), - Kind::Run => describe!( - run::ExpandYamlAnchors, - run::BuildManifest, - run::BumpStage0, - run::ReplaceVersionPlaceholder, - run::Miri, - run::CollectLicenseMetadata, - run::GenerateCopyright, - run::GenerateWindowsSys, - run::GenerateCompletions, - ), - Kind::Setup => describe!(setup::Profile, setup::Hook, setup::Link, setup::Vscode), - Kind::Clean => describe!(clean::CleanAll, clean::Rustc, clean::Std), - // special-cased in Build::build() - Kind::Format | Kind::Suggest => vec![], - } - } - - pub fn get_help(build: &Build, kind: Kind) -> Option { - let step_descriptions = Builder::get_step_descriptions(kind); - if step_descriptions.is_empty() { - return None; - } - - let builder = Self::new_internal(build, kind, vec![]); - let builder = &builder; - // The "build" kind here is just a placeholder, it will be replaced with something else in - // the following statement. - let mut should_run = ShouldRun::new(builder, Kind::Build); - for desc in step_descriptions { - should_run.kind = desc.kind; - should_run = (desc.should_run)(should_run); - } - let mut help = String::from("Available paths:\n"); - let mut add_path = |path: &Path| { - t!(write!(help, " ./x.py {} {}\n", kind.as_str(), path.display())); - }; - for pathset in should_run.paths { - match pathset { - PathSet::Set(set) => { - for path in set { - add_path(&path.path); - } - } - PathSet::Suite(path) => { - add_path(&path.path.join("...")); - } - } - } - Some(help) - } - - fn new_internal(build: &Build, kind: Kind, paths: Vec) -> Builder<'_> { - Builder { - build, - top_stage: build.config.stage, - kind, - cache: Cache::new(), - stack: RefCell::new(Vec::new()), - time_spent_on_dependencies: Cell::new(Duration::new(0, 0)), - paths, - } - } - - pub fn new(build: &Build) -> Builder<'_> { - let paths = &build.config.paths; - let (kind, paths) = match build.config.cmd { - Subcommand::Build => (Kind::Build, &paths[..]), - Subcommand::Check { .. } => (Kind::Check, &paths[..]), - Subcommand::Clippy { .. } => (Kind::Clippy, &paths[..]), - Subcommand::Fix => (Kind::Fix, &paths[..]), - Subcommand::Doc { .. } => (Kind::Doc, &paths[..]), - Subcommand::Test { .. } => (Kind::Test, &paths[..]), - Subcommand::Bench { .. } => (Kind::Bench, &paths[..]), - Subcommand::Dist => (Kind::Dist, &paths[..]), - Subcommand::Install => (Kind::Install, &paths[..]), - Subcommand::Run { .. } => (Kind::Run, &paths[..]), - Subcommand::Clean { .. } => (Kind::Clean, &paths[..]), - Subcommand::Format { .. } => (Kind::Format, &[][..]), - Subcommand::Suggest { .. } => (Kind::Suggest, &[][..]), - Subcommand::Setup { profile: ref path } => ( - Kind::Setup, - path.as_ref().map_or([].as_slice(), |path| std::slice::from_ref(path)), - ), - }; - - Self::new_internal(build, kind, paths.to_owned()) - } - - pub fn execute_cli(&self) { - self.run_step_descriptions(&Builder::get_step_descriptions(self.kind), &self.paths); - } - - pub fn default_doc(&self, paths: &[PathBuf]) { - self.run_step_descriptions(&Builder::get_step_descriptions(Kind::Doc), paths); - } - - pub fn doc_rust_lang_org_channel(&self) -> String { - let channel = match &*self.config.channel { - "stable" => &self.version, - "beta" => "beta", - "nightly" | "dev" => "nightly", - // custom build of rustdoc maybe? link to the latest stable docs just in case - _ => "stable", - }; - "https://doc.rust-lang.org/".to_owned() + channel - } - - fn run_step_descriptions(&self, v: &[StepDescription], paths: &[PathBuf]) { - StepDescription::run(v, self, paths); - } - - /// Obtain a compiler at a given stage and for a given host. Explicitly does - /// not take `Compiler` since all `Compiler` instances are meant to be - /// obtained through this function, since it ensures that they are valid - /// (i.e., built and assembled). - pub fn compiler(&self, stage: u32, host: TargetSelection) -> Compiler { - self.ensure(compile::Assemble { target_compiler: Compiler { stage, host } }) - } - - /// Similar to `compiler`, except handles the full-bootstrap option to - /// silently use the stage1 compiler instead of a stage2 compiler if one is - /// requested. - /// - /// Note that this does *not* have the side effect of creating - /// `compiler(stage, host)`, unlike `compiler` above which does have such - /// a side effect. The returned compiler here can only be used to compile - /// new artifacts, it can't be used to rely on the presence of a particular - /// sysroot. - /// - /// See `force_use_stage1` and `force_use_stage2` for documentation on what each argument is. - pub fn compiler_for( - &self, - stage: u32, - host: TargetSelection, - target: TargetSelection, - ) -> Compiler { - if self.build.force_use_stage2(stage) { - self.compiler(2, self.config.build) - } else if self.build.force_use_stage1(stage, target) { - self.compiler(1, self.config.build) - } else { - self.compiler(stage, host) - } - } - - pub fn sysroot(&self, compiler: Compiler) -> Interned { - self.ensure(compile::Sysroot::new(compiler)) - } - - /// Returns the libdir where the standard library and other artifacts are - /// found for a compiler's sysroot. - pub fn sysroot_libdir(&self, compiler: Compiler, target: TargetSelection) -> Interned { - #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] - struct Libdir { - compiler: Compiler, - target: TargetSelection, - } - impl Step for Libdir { - type Output = Interned; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - fn run(self, builder: &Builder<'_>) -> Interned { - let lib = builder.sysroot_libdir_relative(self.compiler); - let sysroot = builder - .sysroot(self.compiler) - .join(lib) - .join("rustlib") - .join(self.target.triple) - .join("lib"); - // Avoid deleting the rustlib/ directory we just copied - // (in `impl Step for Sysroot`). - if !builder.download_rustc() { - builder.verbose(&format!( - "Removing sysroot {} to avoid caching bugs", - sysroot.display() - )); - let _ = fs::remove_dir_all(&sysroot); - t!(fs::create_dir_all(&sysroot)); - } - - if self.compiler.stage == 0 { - // The stage 0 compiler for the build triple is always pre-built. - // Ensure that `libLLVM.so` ends up in the target libdir, so that ui-fulldeps tests can use it when run. - dist::maybe_install_llvm_target( - builder, - self.compiler.host, - &builder.sysroot(self.compiler), - ); - } - - INTERNER.intern_path(sysroot) - } - } - self.ensure(Libdir { compiler, target }) - } - - pub fn sysroot_codegen_backends(&self, compiler: Compiler) -> PathBuf { - self.sysroot_libdir(compiler, compiler.host).with_file_name("codegen-backends") - } - - /// Returns the compiler's libdir where it stores the dynamic libraries that - /// it itself links against. - /// - /// For example this returns `/lib` on Unix and `/bin` on - /// Windows. - pub fn rustc_libdir(&self, compiler: Compiler) -> PathBuf { - if compiler.is_snapshot(self) { - self.rustc_snapshot_libdir() - } else { - match self.config.libdir_relative() { - Some(relative_libdir) if compiler.stage >= 1 => { - self.sysroot(compiler).join(relative_libdir) - } - _ => self.sysroot(compiler).join(libdir(compiler.host)), - } - } - } - - /// Returns the compiler's relative libdir where it stores the dynamic libraries that - /// it itself links against. - /// - /// For example this returns `lib` on Unix and `bin` on - /// Windows. - pub fn libdir_relative(&self, compiler: Compiler) -> &Path { - if compiler.is_snapshot(self) { - libdir(self.config.build).as_ref() - } else { - match self.config.libdir_relative() { - Some(relative_libdir) if compiler.stage >= 1 => relative_libdir, - _ => libdir(compiler.host).as_ref(), - } - } - } - - /// Returns the compiler's relative libdir where the standard library and other artifacts are - /// found for a compiler's sysroot. - /// - /// For example this returns `lib` on Unix and Windows. - pub fn sysroot_libdir_relative(&self, compiler: Compiler) -> &Path { - match self.config.libdir_relative() { - Some(relative_libdir) if compiler.stage >= 1 => relative_libdir, - _ if compiler.stage == 0 => &self.build.initial_libdir, - _ => Path::new("lib"), - } - } - - pub fn rustc_lib_paths(&self, compiler: Compiler) -> Vec { - let mut dylib_dirs = vec![self.rustc_libdir(compiler)]; - - // Ensure that the downloaded LLVM libraries can be found. - if self.config.llvm_from_ci { - let ci_llvm_lib = self.out.join(&*compiler.host.triple).join("ci-llvm").join("lib"); - dylib_dirs.push(ci_llvm_lib); - } - - dylib_dirs - } - - /// Adds the compiler's directory of dynamic libraries to `cmd`'s dynamic - /// library lookup path. - pub fn add_rustc_lib_path(&self, compiler: Compiler, cmd: &mut Command) { - // Windows doesn't need dylib path munging because the dlls for the - // compiler live next to the compiler and the system will find them - // automatically. - if cfg!(windows) { - return; - } - - add_dylib_path(self.rustc_lib_paths(compiler), cmd); - } - - /// Gets a path to the compiler specified. - pub fn rustc(&self, compiler: Compiler) -> PathBuf { - if compiler.is_snapshot(self) { - self.initial_rustc.clone() - } else { - self.sysroot(compiler).join("bin").join(exe("rustc", compiler.host)) - } - } - - /// Gets the paths to all of the compiler's codegen backends. - fn codegen_backends(&self, compiler: Compiler) -> impl Iterator { - fs::read_dir(self.sysroot_codegen_backends(compiler)) - .into_iter() - .flatten() - .filter_map(Result::ok) - .map(|entry| entry.path()) - } - - pub fn rustdoc(&self, compiler: Compiler) -> PathBuf { - self.ensure(tool::Rustdoc { compiler }) - } - - pub fn rustdoc_cmd(&self, compiler: Compiler) -> Command { - let mut cmd = Command::new(&self.bootstrap_out.join("rustdoc")); - cmd.env("RUSTC_STAGE", compiler.stage.to_string()) - .env("RUSTC_SYSROOT", self.sysroot(compiler)) - // Note that this is *not* the sysroot_libdir because rustdoc must be linked - // equivalently to rustc. - .env("RUSTDOC_LIBDIR", self.rustc_libdir(compiler)) - .env("CFG_RELEASE_CHANNEL", &self.config.channel) - .env("RUSTDOC_REAL", self.rustdoc(compiler)) - .env("RUSTC_BOOTSTRAP", "1"); - - cmd.arg("-Wrustdoc::invalid_codeblock_attributes"); - - if self.config.deny_warnings { - cmd.arg("-Dwarnings"); - } - cmd.arg("-Znormalize-docs"); - - // Remove make-related flags that can cause jobserver problems. - cmd.env_remove("MAKEFLAGS"); - cmd.env_remove("MFLAGS"); - - if let Some(linker) = self.linker(compiler.host) { - cmd.env("RUSTDOC_LINKER", linker); - } - if self.is_fuse_ld_lld(compiler.host) { - cmd.env("RUSTDOC_FUSE_LD_LLD", "1"); - } - cmd - } - - /// Return the path to `llvm-config` for the target, if it exists. - /// - /// Note that this returns `None` if LLVM is disabled, or if we're in a - /// check build or dry-run, where there's no need to build all of LLVM. - fn llvm_config(&self, target: TargetSelection) -> Option { - if self.config.llvm_enabled() && self.kind != Kind::Check && !self.config.dry_run() { - let llvm::LlvmResult { llvm_config, .. } = self.ensure(llvm::Llvm { target }); - if llvm_config.is_file() { - return Some(llvm_config); - } - } - None - } - - /// Like `cargo`, but only passes flags that are valid for all commands. - pub fn bare_cargo( - &self, - compiler: Compiler, - mode: Mode, - target: TargetSelection, - cmd: &str, - ) -> Command { - let mut cargo = Command::new(&self.initial_cargo); - // Run cargo from the source root so it can find .cargo/config. - // This matters when using vendoring and the working directory is outside the repository. - cargo.current_dir(&self.src); - - let out_dir = self.stage_out(compiler, mode); - cargo.env("CARGO_TARGET_DIR", &out_dir).arg(cmd); - - // Found with `rg "init_env_logger\("`. If anyone uses `init_env_logger` - // from out of tree it shouldn't matter, since x.py is only used for - // building in-tree. - let color_logs = ["RUSTDOC_LOG_COLOR", "RUSTC_LOG_COLOR", "RUST_LOG_COLOR"]; - match self.build.config.color { - Color::Always => { - cargo.arg("--color=always"); - for log in &color_logs { - cargo.env(log, "always"); - } - } - Color::Never => { - cargo.arg("--color=never"); - for log in &color_logs { - cargo.env(log, "never"); - } - } - Color::Auto => {} // nothing to do - } - - if cmd != "install" { - cargo.arg("--target").arg(target.rustc_target_arg()); - } else { - assert_eq!(target, compiler.host); - } - - if self.config.rust_optimize.is_release() { - // FIXME: cargo bench/install do not accept `--release` - if cmd != "bench" && cmd != "install" { - cargo.arg("--release"); - } - } - - // Remove make-related flags to ensure Cargo can correctly set things up - cargo.env_remove("MAKEFLAGS"); - cargo.env_remove("MFLAGS"); - - cargo - } - - /// Prepares an invocation of `cargo` to be run. - /// - /// This will create a `Command` that represents a pending execution of - /// Cargo. This cargo will be configured to use `compiler` as the actual - /// rustc compiler, its output will be scoped by `mode`'s output directory, - /// it will pass the `--target` flag for the specified `target`, and will be - /// executing the Cargo command `cmd`. - pub fn cargo( - &self, - compiler: Compiler, - mode: Mode, - source_type: SourceType, - target: TargetSelection, - cmd: &str, - ) -> Cargo { - let mut cargo = self.bare_cargo(compiler, mode, target, cmd); - let out_dir = self.stage_out(compiler, mode); - - // Codegen backends are not yet tracked by -Zbinary-dep-depinfo, - // so we need to explicitly clear out if they've been updated. - for backend in self.codegen_backends(compiler) { - self.clear_if_dirty(&out_dir, &backend); - } - - if cmd == "doc" || cmd == "rustdoc" { - let my_out = match mode { - // This is the intended out directory for compiler documentation. - Mode::Rustc | Mode::ToolRustc => self.compiler_doc_out(target), - Mode::Std => { - if self.config.cmd.json() { - out_dir.join(target.triple).join("json-doc") - } else { - out_dir.join(target.triple).join("doc") - } - } - _ => panic!("doc mode {mode:?} not expected"), - }; - let rustdoc = self.rustdoc(compiler); - self.clear_if_dirty(&my_out, &rustdoc); - } - - let profile_var = |name: &str| { - let profile = if self.config.rust_optimize.is_release() { "RELEASE" } else { "DEV" }; - format!("CARGO_PROFILE_{}_{}", profile, name) - }; - - // See comment in rustc_llvm/build.rs for why this is necessary, largely llvm-config - // needs to not accidentally link to libLLVM in stage0/lib. - cargo.env("REAL_LIBRARY_PATH_VAR", &util::dylib_path_var()); - if let Some(e) = env::var_os(util::dylib_path_var()) { - cargo.env("REAL_LIBRARY_PATH", e); - } - - // Set a flag for `check`/`clippy`/`fix`, so that certain build - // scripts can do less work (i.e. not building/requiring LLVM). - if cmd == "check" || cmd == "clippy" || cmd == "fix" { - // If we've not yet built LLVM, or it's stale, then bust - // the rustc_llvm cache. That will always work, even though it - // may mean that on the next non-check build we'll need to rebuild - // rustc_llvm. But if LLVM is stale, that'll be a tiny amount - // of work comparatively, and we'd likely need to rebuild it anyway, - // so that's okay. - if crate::llvm::prebuilt_llvm_config(self, target).is_err() { - cargo.env("RUST_CHECK", "1"); - } - } - - let stage = if compiler.stage == 0 && self.local_rebuild { - // Assume the local-rebuild rustc already has stage1 features. - 1 - } else { - compiler.stage - }; - - let mut rustflags = Rustflags::new(target); - if stage != 0 { - if let Ok(s) = env::var("CARGOFLAGS_NOT_BOOTSTRAP") { - cargo.args(s.split_whitespace()); - } - rustflags.env("RUSTFLAGS_NOT_BOOTSTRAP"); - } else { - if let Ok(s) = env::var("CARGOFLAGS_BOOTSTRAP") { - cargo.args(s.split_whitespace()); - } - rustflags.env("RUSTFLAGS_BOOTSTRAP"); - if cmd == "clippy" { - // clippy overwrites sysroot if we pass it to cargo. - // Pass it directly to clippy instead. - // NOTE: this can't be fixed in clippy because we explicitly don't set `RUSTC`, - // so it has no way of knowing the sysroot. - rustflags.arg("--sysroot"); - rustflags.arg( - self.sysroot(compiler) - .as_os_str() - .to_str() - .expect("sysroot must be valid UTF-8"), - ); - // Only run clippy on a very limited subset of crates (in particular, not build scripts). - cargo.arg("-Zunstable-options"); - // Explicitly does *not* set `--cfg=bootstrap`, since we're using a nightly clippy. - let host_version = Command::new("rustc").arg("--version").output().map_err(|_| ()); - let output = host_version.and_then(|output| { - if output.status.success() { - Ok(output) - } else { - Err(()) - } - }).unwrap_or_else(|_| { - eprintln!( - "error: `x.py clippy` requires a host `rustc` toolchain with the `clippy` component" - ); - eprintln!("help: try `rustup component add clippy`"); - crate::exit!(1); - }); - if !t!(std::str::from_utf8(&output.stdout)).contains("nightly") { - rustflags.arg("--cfg=bootstrap"); - } - } else { - rustflags.arg("--cfg=bootstrap"); - } - } - - let use_new_symbol_mangling = match self.config.rust_new_symbol_mangling { - Some(setting) => { - // If an explicit setting is given, use that - setting - } - None => { - if mode == Mode::Std { - // The standard library defaults to the legacy scheme - false - } else { - // The compiler and tools default to the new scheme - true - } - } - }; - - // By default, windows-rs depends on a native library that doesn't get copied into the - // sysroot. Passing this cfg enables raw-dylib support instead, which makes the native - // library unnecessary. This can be removed when windows-rs enables raw-dylib - // unconditionally. - if let Mode::Rustc | Mode::ToolRustc = mode { - rustflags.arg("--cfg=windows_raw_dylib"); - } - - if use_new_symbol_mangling { - rustflags.arg("-Csymbol-mangling-version=v0"); - } else { - rustflags.arg("-Csymbol-mangling-version=legacy"); - rustflags.arg("-Zunstable-options"); - } - - // Enable cfg checking of cargo features for everything but std and also enable cfg - // checking of names and values. - // - // Note: `std`, `alloc` and `core` imports some dependencies by #[path] (like - // backtrace, core_simd, std_float, ...), those dependencies have their own - // features but cargo isn't involved in the #[path] process and so cannot pass the - // complete list of features, so for that reason we don't enable checking of - // features for std crates. - cargo.arg(if mode != Mode::Std { - "-Zcheck-cfg=names,values,output,features" - } else { - "-Zcheck-cfg=names,values,output" - }); - - // Add extra cfg not defined in/by rustc - // - // Note: Although it would seems that "-Zunstable-options" to `rustflags` is useless as - // cargo would implicitly add it, it was discover that sometimes bootstrap only use - // `rustflags` without `cargo` making it required. - rustflags.arg("-Zunstable-options"); - for (restricted_mode, name, values) in EXTRA_CHECK_CFGS { - if *restricted_mode == None || *restricted_mode == Some(mode) { - // Creating a string of the values by concatenating each value: - // ',"tvos","watchos"' or '' (nothing) when there are no values - let values = match values { - Some(values) => values - .iter() - .map(|val| [",", "\"", val, "\""]) - .flatten() - .collect::(), - None => String::new(), - }; - rustflags.arg(&format!("--check-cfg=values({name}{values})")); - } - } - - // FIXME: It might be better to use the same value for both `RUSTFLAGS` and `RUSTDOCFLAGS`, - // but this breaks CI. At the very least, stage0 `rustdoc` needs `--cfg bootstrap`. See - // #71458. - let mut rustdocflags = rustflags.clone(); - rustdocflags.propagate_cargo_env("RUSTDOCFLAGS"); - if stage == 0 { - rustdocflags.env("RUSTDOCFLAGS_BOOTSTRAP"); - } else { - rustdocflags.env("RUSTDOCFLAGS_NOT_BOOTSTRAP"); - } - - if let Ok(s) = env::var("CARGOFLAGS") { - cargo.args(s.split_whitespace()); - } - - match mode { - Mode::Std | Mode::ToolBootstrap | Mode::ToolStd => {} - Mode::Rustc | Mode::Codegen | Mode::ToolRustc => { - // Build proc macros both for the host and the target - if target != compiler.host && cmd != "check" { - cargo.arg("-Zdual-proc-macros"); - rustflags.arg("-Zdual-proc-macros"); - } - } - } - - // This tells Cargo (and in turn, rustc) to output more complete - // dependency information. Most importantly for rustbuild, this - // includes sysroot artifacts, like libstd, which means that we don't - // need to track those in rustbuild (an error prone process!). This - // feature is currently unstable as there may be some bugs and such, but - // it represents a big improvement in rustbuild's reliability on - // rebuilds, so we're using it here. - // - // For some additional context, see #63470 (the PR originally adding - // this), as well as #63012 which is the tracking issue for this - // feature on the rustc side. - cargo.arg("-Zbinary-dep-depinfo"); - let allow_features = match mode { - Mode::ToolBootstrap | Mode::ToolStd => { - // Restrict the allowed features so we don't depend on nightly - // accidentally. - // - // binary-dep-depinfo is used by rustbuild itself for all - // compilations. - // - // Lots of tools depend on proc_macro2 and proc-macro-error. - // Those have build scripts which assume nightly features are - // available if the `rustc` version is "nighty" or "dev". See - // bin/rustc.rs for why that is a problem. Instead of labeling - // those features for each individual tool that needs them, - // just blanket allow them here. - // - // If this is ever removed, be sure to add something else in - // its place to keep the restrictions in place (or make a way - // to unset RUSTC_BOOTSTRAP). - "binary-dep-depinfo,proc_macro_span,proc_macro_span_shrink,proc_macro_diagnostic" - .to_string() - } - Mode::Std | Mode::Rustc | Mode::Codegen | Mode::ToolRustc => String::new(), - }; - - cargo.arg("-j").arg(self.jobs().to_string()); - - // FIXME: Temporary fix for https://github.com/rust-lang/cargo/issues/3005 - // Force cargo to output binaries with disambiguating hashes in the name - let mut metadata = if compiler.stage == 0 { - // Treat stage0 like a special channel, whether it's a normal prior- - // release rustc or a local rebuild with the same version, so we - // never mix these libraries by accident. - "bootstrap".to_string() - } else { - self.config.channel.to_string() - }; - // We want to make sure that none of the dependencies between - // std/test/rustc unify with one another. This is done for weird linkage - // reasons but the gist of the problem is that if librustc, libtest, and - // libstd all depend on libc from crates.io (which they actually do) we - // want to make sure they all get distinct versions. Things get really - // weird if we try to unify all these dependencies right now, namely - // around how many times the library is linked in dynamic libraries and - // such. If rustc were a static executable or if we didn't ship dylibs - // this wouldn't be a problem, but we do, so it is. This is in general - // just here to make sure things build right. If you can remove this and - // things still build right, please do! - match mode { - Mode::Std => metadata.push_str("std"), - // When we're building rustc tools, they're built with a search path - // that contains things built during the rustc build. For example, - // bitflags is built during the rustc build, and is a dependency of - // rustdoc as well. We're building rustdoc in a different target - // directory, though, which means that Cargo will rebuild the - // dependency. When we go on to build rustdoc, we'll look for - // bitflags, and find two different copies: one built during the - // rustc step and one that we just built. This isn't always a - // problem, somehow -- not really clear why -- but we know that this - // fixes things. - Mode::ToolRustc => metadata.push_str("tool-rustc"), - // Same for codegen backends. - Mode::Codegen => metadata.push_str("codegen"), - _ => {} - } - cargo.env("__CARGO_DEFAULT_LIB_METADATA", &metadata); - - if cmd == "clippy" { - rustflags.arg("-Zforce-unstable-if-unmarked"); - } - - rustflags.arg("-Zmacro-backtrace"); - - let want_rustdoc = self.doc_tests != DocTests::No; - - // We synthetically interpret a stage0 compiler used to build tools as a - // "raw" compiler in that it's the exact snapshot we download. Normally - // the stage0 build means it uses libraries build by the stage0 - // compiler, but for tools we just use the precompiled libraries that - // we've downloaded - let use_snapshot = mode == Mode::ToolBootstrap; - assert!(!use_snapshot || stage == 0 || self.local_rebuild); - - let maybe_sysroot = self.sysroot(compiler); - let sysroot = if use_snapshot { self.rustc_snapshot_sysroot() } else { &maybe_sysroot }; - let libdir = self.rustc_libdir(compiler); - - // Clear the output directory if the real rustc we're using has changed; - // Cargo cannot detect this as it thinks rustc is bootstrap/debug/rustc. - // - // Avoid doing this during dry run as that usually means the relevant - // compiler is not yet linked/copied properly. - // - // Only clear out the directory if we're compiling std; otherwise, we - // should let Cargo take care of things for us (via depdep info) - if !self.config.dry_run() && mode == Mode::Std && cmd == "build" { - self.clear_if_dirty(&out_dir, &self.rustc(compiler)); - } - - // Customize the compiler we're running. Specify the compiler to cargo - // as our shim and then pass it some various options used to configure - // how the actual compiler itself is called. - // - // These variables are primarily all read by - // src/bootstrap/bin/{rustc.rs,rustdoc.rs} - cargo - .env("RUSTBUILD_NATIVE_DIR", self.native_dir(target)) - .env("RUSTC_REAL", self.rustc(compiler)) - .env("RUSTC_STAGE", stage.to_string()) - .env("RUSTC_SYSROOT", &sysroot) - .env("RUSTC_LIBDIR", &libdir) - .env("RUSTDOC", self.bootstrap_out.join("rustdoc")) - .env( - "RUSTDOC_REAL", - if cmd == "doc" || cmd == "rustdoc" || (cmd == "test" && want_rustdoc) { - self.rustdoc(compiler) - } else { - PathBuf::from("/path/to/nowhere/rustdoc/not/required") - }, - ) - .env("RUSTC_ERROR_METADATA_DST", self.extended_error_dir()) - .env("RUSTC_BREAK_ON_ICE", "1"); - // Clippy support is a hack and uses the default `cargo-clippy` in path. - // Don't override RUSTC so that the `cargo-clippy` in path will be run. - if cmd != "clippy" { - cargo.env("RUSTC", self.bootstrap_out.join("rustc")); - } - - // Dealing with rpath here is a little special, so let's go into some - // detail. First off, `-rpath` is a linker option on Unix platforms - // which adds to the runtime dynamic loader path when looking for - // dynamic libraries. We use this by default on Unix platforms to ensure - // that our nightlies behave the same on Windows, that is they work out - // of the box. This can be disabled by setting `rpath = false` in `[rust]` - // table of `config.toml` - // - // Ok, so the astute might be wondering "why isn't `-C rpath` used - // here?" and that is indeed a good question to ask. This codegen - // option is the compiler's current interface to generating an rpath. - // Unfortunately it doesn't quite suffice for us. The flag currently - // takes no value as an argument, so the compiler calculates what it - // should pass to the linker as `-rpath`. This unfortunately is based on - // the **compile time** directory structure which when building with - // Cargo will be very different than the runtime directory structure. - // - // All that's a really long winded way of saying that if we use - // `-Crpath` then the executables generated have the wrong rpath of - // something like `$ORIGIN/deps` when in fact the way we distribute - // rustc requires the rpath to be `$ORIGIN/../lib`. - // - // So, all in all, to set up the correct rpath we pass the linker - // argument manually via `-C link-args=-Wl,-rpath,...`. Plus isn't it - // fun to pass a flag to a tool to pass a flag to pass a flag to a tool - // to change a flag in a binary? - if self.config.rpath_enabled(target) && util::use_host_linker(target) { - let libdir = self.sysroot_libdir_relative(compiler).to_str().unwrap(); - let rpath = if target.contains("apple") { - // Note that we need to take one extra step on macOS to also pass - // `-Wl,-instal_name,@rpath/...` to get things to work right. To - // do that we pass a weird flag to the compiler to get it to do - // so. Note that this is definitely a hack, and we should likely - // flesh out rpath support more fully in the future. - rustflags.arg("-Zosx-rpath-install-name"); - Some(format!("-Wl,-rpath,@loader_path/../{libdir}")) - } else if !target.contains("windows") - && !target.contains("aix") - && !target.contains("xous") - { - rustflags.arg("-Clink-args=-Wl,-z,origin"); - Some(format!("-Wl,-rpath,$ORIGIN/../{libdir}")) - } else { - None - }; - if let Some(rpath) = rpath { - rustflags.arg(&format!("-Clink-args={rpath}")); - } - } - - if let Some(host_linker) = self.linker(compiler.host) { - cargo.env("RUSTC_HOST_LINKER", host_linker); - } - if self.is_fuse_ld_lld(compiler.host) { - cargo.env("RUSTC_HOST_FUSE_LD_LLD", "1"); - cargo.env("RUSTDOC_FUSE_LD_LLD", "1"); - } - - if let Some(target_linker) = self.linker(target) { - let target = crate::envify(&target.triple); - cargo.env(&format!("CARGO_TARGET_{target}_LINKER"), target_linker); - } - if self.is_fuse_ld_lld(target) { - rustflags.arg("-Clink-args=-fuse-ld=lld"); - } - self.lld_flags(target).for_each(|flag| { - rustdocflags.arg(&flag); - }); - - if !(["build", "check", "clippy", "fix", "rustc"].contains(&cmd)) && want_rustdoc { - cargo.env("RUSTDOC_LIBDIR", self.rustc_libdir(compiler)); - } - - let debuginfo_level = match mode { - Mode::Rustc | Mode::Codegen => self.config.rust_debuginfo_level_rustc, - Mode::Std => self.config.rust_debuginfo_level_std, - Mode::ToolBootstrap | Mode::ToolStd | Mode::ToolRustc => { - self.config.rust_debuginfo_level_tools - } - }; - cargo.env(profile_var("DEBUG"), debuginfo_level.to_string()); - if let Some(opt_level) = &self.config.rust_optimize.get_opt_level() { - cargo.env(profile_var("OPT_LEVEL"), opt_level); - } - if !self.config.dry_run() && self.cc.borrow()[&target].args().iter().any(|arg| arg == "-gz") - { - rustflags.arg("-Clink-arg=-gz"); - } - cargo.env( - profile_var("DEBUG_ASSERTIONS"), - if mode == Mode::Std { - self.config.rust_debug_assertions_std.to_string() - } else { - self.config.rust_debug_assertions.to_string() - }, - ); - cargo.env( - profile_var("OVERFLOW_CHECKS"), - if mode == Mode::Std { - self.config.rust_overflow_checks_std.to_string() - } else { - self.config.rust_overflow_checks.to_string() - }, - ); - - let split_debuginfo_is_stable = target.contains("linux") - || target.contains("apple") - || (target.contains("msvc") - && self.config.rust_split_debuginfo == SplitDebuginfo::Packed) - || (target.contains("windows") - && self.config.rust_split_debuginfo == SplitDebuginfo::Off); - - if !split_debuginfo_is_stable { - rustflags.arg("-Zunstable-options"); - } - match self.config.rust_split_debuginfo { - SplitDebuginfo::Packed => rustflags.arg("-Csplit-debuginfo=packed"), - SplitDebuginfo::Unpacked => rustflags.arg("-Csplit-debuginfo=unpacked"), - SplitDebuginfo::Off => rustflags.arg("-Csplit-debuginfo=off"), - }; - - if self.config.cmd.bless() { - // Bless `expect!` tests. - cargo.env("UPDATE_EXPECT", "1"); - } - - if !mode.is_tool() { - cargo.env("RUSTC_FORCE_UNSTABLE", "1"); - } - - if let Some(x) = self.crt_static(target) { - if x { - rustflags.arg("-Ctarget-feature=+crt-static"); - } else { - rustflags.arg("-Ctarget-feature=-crt-static"); - } - } - - if let Some(x) = self.crt_static(compiler.host) { - cargo.env("RUSTC_HOST_CRT_STATIC", x.to_string()); - } - - if let Some(map_to) = self.build.debuginfo_map_to(GitRepo::Rustc) { - let map = format!("{}={}", self.build.src.display(), map_to); - cargo.env("RUSTC_DEBUGINFO_MAP", map); - - // `rustc` needs to know the virtual `/rustc/$hash` we're mapping to, - // in order to opportunistically reverse it later. - cargo.env("CFG_VIRTUAL_RUST_SOURCE_BASE_DIR", map_to); - } - - // Enable usage of unstable features - cargo.env("RUSTC_BOOTSTRAP", "1"); - self.add_rust_test_threads(&mut cargo); - - // Almost all of the crates that we compile as part of the bootstrap may - // have a build script, including the standard library. To compile a - // build script, however, it itself needs a standard library! This - // introduces a bit of a pickle when we're compiling the standard - // library itself. - // - // To work around this we actually end up using the snapshot compiler - // (stage0) for compiling build scripts of the standard library itself. - // The stage0 compiler is guaranteed to have a libstd available for use. - // - // For other crates, however, we know that we've already got a standard - // library up and running, so we can use the normal compiler to compile - // build scripts in that situation. - if mode == Mode::Std { - cargo - .env("RUSTC_SNAPSHOT", &self.initial_rustc) - .env("RUSTC_SNAPSHOT_LIBDIR", self.rustc_snapshot_libdir()); - } else { - cargo - .env("RUSTC_SNAPSHOT", self.rustc(compiler)) - .env("RUSTC_SNAPSHOT_LIBDIR", self.rustc_libdir(compiler)); - } - - // Tools that use compiler libraries may inherit the `-lLLVM` link - // requirement, but the `-L` library path is not propagated across - // separate Cargo projects. We can add LLVM's library path to the - // platform-specific environment variable as a workaround. - if mode == Mode::ToolRustc || mode == Mode::Codegen { - if let Some(llvm_config) = self.llvm_config(target) { - let llvm_libdir = output(Command::new(&llvm_config).arg("--libdir")); - add_link_lib_path(vec![llvm_libdir.trim().into()], &mut cargo); - } - } - - // Compile everything except libraries and proc macros with the more - // efficient initial-exec TLS model. This doesn't work with `dlopen`, - // so we can't use it by default in general, but we can use it for tools - // and our own internal libraries. - if !mode.must_support_dlopen() && !target.triple.starts_with("powerpc-") { - cargo.env("RUSTC_TLS_MODEL_INITIAL_EXEC", "1"); - } - - // Ignore incremental modes except for stage0, since we're - // not guaranteeing correctness across builds if the compiler - // is changing under your feet. - if self.config.incremental && compiler.stage == 0 { - cargo.env("CARGO_INCREMENTAL", "1"); - } else { - // Don't rely on any default setting for incr. comp. in Cargo - cargo.env("CARGO_INCREMENTAL", "0"); - } - - if let Some(ref on_fail) = self.config.on_fail { - cargo.env("RUSTC_ON_FAIL", on_fail); - } - - if self.config.print_step_timings { - cargo.env("RUSTC_PRINT_STEP_TIMINGS", "1"); - } - - if self.config.print_step_rusage { - cargo.env("RUSTC_PRINT_STEP_RUSAGE", "1"); - } - - if self.config.backtrace_on_ice { - cargo.env("RUSTC_BACKTRACE_ON_ICE", "1"); - } - - cargo.env("RUSTC_VERBOSE", self.verbosity.to_string()); - - // Downstream forks of the Rust compiler might want to use a custom libc to add support for - // targets that are not yet available upstream. Adding a patch to replace libc with a - // custom one would cause compilation errors though, because Cargo would interpret the - // custom libc as part of the workspace, and apply the check-cfg lints on it. - // - // The libc build script emits check-cfg flags only when this environment variable is set, - // so this line allows the use of custom libcs. - cargo.env("LIBC_CHECK_CFG", "1"); - - if source_type == SourceType::InTree { - let mut lint_flags = Vec::new(); - // When extending this list, add the new lints to the RUSTFLAGS of the - // build_bootstrap function of src/bootstrap/bootstrap.py as well as - // some code doesn't go through this `rustc` wrapper. - lint_flags.push("-Wrust_2018_idioms"); - lint_flags.push("-Wunused_lifetimes"); - lint_flags.push("-Wsemicolon_in_expressions_from_macros"); - - if self.config.deny_warnings { - lint_flags.push("-Dwarnings"); - rustdocflags.arg("-Dwarnings"); - } - - // This does not use RUSTFLAGS due to caching issues with Cargo. - // Clippy is treated as an "in tree" tool, but shares the same - // cache as other "submodule" tools. With these options set in - // RUSTFLAGS, that causes *every* shared dependency to be rebuilt. - // By injecting this into the rustc wrapper, this circumvents - // Cargo's fingerprint detection. This is fine because lint flags - // are always ignored in dependencies. Eventually this should be - // fixed via better support from Cargo. - cargo.env("RUSTC_LINT_FLAGS", lint_flags.join(" ")); - - rustdocflags.arg("-Wrustdoc::invalid_codeblock_attributes"); - } - - if mode == Mode::Rustc { - rustflags.arg("-Zunstable-options"); - rustflags.arg("-Wrustc::internal"); - } - - // Throughout the build Cargo can execute a number of build scripts - // compiling C/C++ code and we need to pass compilers, archivers, flags, etc - // obtained previously to those build scripts. - // Build scripts use either the `cc` crate or `configure/make` so we pass - // the options through environment variables that are fetched and understood by both. - // - // FIXME: the guard against msvc shouldn't need to be here - if target.contains("msvc") { - if let Some(ref cl) = self.config.llvm_clang_cl { - cargo.env("CC", cl).env("CXX", cl); - } - } else { - let ccache = self.config.ccache.as_ref(); - let ccacheify = |s: &Path| { - let ccache = match ccache { - Some(ref s) => s, - None => return s.display().to_string(), - }; - // FIXME: the cc-rs crate only recognizes the literal strings - // `ccache` and `sccache` when doing caching compilations, so we - // mirror that here. It should probably be fixed upstream to - // accept a new env var or otherwise work with custom ccache - // vars. - match &ccache[..] { - "ccache" | "sccache" => format!("{} {}", ccache, s.display()), - _ => s.display().to_string(), - } - }; - let triple_underscored = target.triple.replace("-", "_"); - let cc = ccacheify(&self.cc(target)); - cargo.env(format!("CC_{triple_underscored}"), &cc); - - let cflags = self.cflags(target, GitRepo::Rustc, CLang::C).join(" "); - cargo.env(format!("CFLAGS_{triple_underscored}"), &cflags); - - if let Some(ar) = self.ar(target) { - let ranlib = format!("{} s", ar.display()); - cargo - .env(format!("AR_{triple_underscored}"), ar) - .env(format!("RANLIB_{triple_underscored}"), ranlib); - } - - if let Ok(cxx) = self.cxx(target) { - let cxx = ccacheify(&cxx); - let cxxflags = self.cflags(target, GitRepo::Rustc, CLang::Cxx).join(" "); - cargo - .env(format!("CXX_{triple_underscored}"), &cxx) - .env(format!("CXXFLAGS_{triple_underscored}"), cxxflags); - } - } - - // If Control Flow Guard is enabled, pass the `control-flow-guard` flag to rustc - // when compiling the standard library, since this might be linked into the final outputs - // produced by rustc. Since this mitigation is only available on Windows, only enable it - // for the standard library in case the compiler is run on a non-Windows platform. - // This is not needed for stage 0 artifacts because these will only be used for building - // the stage 1 compiler. - if cfg!(windows) - && mode == Mode::Std - && self.config.control_flow_guard - && compiler.stage >= 1 - { - rustflags.arg("-Ccontrol-flow-guard"); - } - - // For `cargo doc` invocations, make rustdoc print the Rust version into the docs - // This replaces spaces with tabs because RUSTDOCFLAGS does not - // support arguments with regular spaces. Hopefully someday Cargo will - // have space support. - let rust_version = self.rust_version().replace(' ', "\t"); - rustdocflags.arg("--crate-version").arg(&rust_version); - - // Environment variables *required* throughout the build - // - // FIXME: should update code to not require this env var - cargo.env("CFG_COMPILER_HOST_TRIPLE", target.triple); - - // Set this for all builds to make sure doc builds also get it. - cargo.env("CFG_RELEASE_CHANNEL", &self.config.channel); - - // This one's a bit tricky. As of the time of this writing the compiler - // links to the `winapi` crate on crates.io. This crate provides raw - // bindings to Windows system functions, sort of like libc does for - // Unix. This crate also, however, provides "import libraries" for the - // MinGW targets. There's an import library per dll in the windows - // distribution which is what's linked to. These custom import libraries - // are used because the winapi crate can reference Windows functions not - // present in the MinGW import libraries. - // - // For example MinGW may ship libdbghelp.a, but it may not have - // references to all the functions in the dbghelp dll. Instead the - // custom import library for dbghelp in the winapi crates has all this - // information. - // - // Unfortunately for us though the import libraries are linked by - // default via `-ldylib=winapi_foo`. That is, they're linked with the - // `dylib` type with a `winapi_` prefix (so the winapi ones don't - // conflict with the system MinGW ones). This consequently means that - // the binaries we ship of things like rustc_codegen_llvm (aka the rustc_codegen_llvm - // DLL) when linked against *again*, for example with procedural macros - // or plugins, will trigger the propagation logic of `-ldylib`, passing - // `-lwinapi_foo` to the linker again. This isn't actually available in - // our distribution, however, so the link fails. - // - // To solve this problem we tell winapi to not use its bundled import - // libraries. This means that it will link to the system MinGW import - // libraries by default, and the `-ldylib=foo` directives will still get - // passed to the final linker, but they'll look like `-lfoo` which can - // be resolved because MinGW has the import library. The downside is we - // don't get newer functions from Windows, but we don't use any of them - // anyway. - if !mode.is_tool() { - cargo.env("WINAPI_NO_BUNDLED_LIBRARIES", "1"); - } - - for _ in 0..self.verbosity { - cargo.arg("-v"); - } - - match (mode, self.config.rust_codegen_units_std, self.config.rust_codegen_units) { - (Mode::Std, Some(n), _) | (_, _, Some(n)) => { - cargo.env(profile_var("CODEGEN_UNITS"), n.to_string()); - } - _ => { - // Don't set anything - } - } - - if self.config.locked_deps { - cargo.arg("--locked"); - } - if self.config.vendor || self.is_sudo { - cargo.arg("--frozen"); - } - - // Try to use a sysroot-relative bindir, in case it was configured absolutely. - cargo.env("RUSTC_INSTALL_BINDIR", self.config.bindir_relative()); - - self.ci_env.force_coloring_in_ci(&mut cargo); - - // When we build Rust dylibs they're all intended for intermediate - // usage, so make sure we pass the -Cprefer-dynamic flag instead of - // linking all deps statically into the dylib. - if matches!(mode, Mode::Std | Mode::Rustc) { - rustflags.arg("-Cprefer-dynamic"); - } - - // When building incrementally we default to a lower ThinLTO import limit - // (unless explicitly specified otherwise). This will produce a somewhat - // slower code but give way better compile times. - { - let limit = match self.config.rust_thin_lto_import_instr_limit { - Some(limit) => Some(limit), - None if self.config.incremental => Some(10), - _ => None, - }; - - if let Some(limit) = limit { - if stage == 0 || self.config.default_codegen_backend().unwrap_or_default() == "llvm" - { - rustflags.arg(&format!("-Cllvm-args=-import-instr-limit={limit}")); - } - } - } - - if matches!(mode, Mode::Std) { - if let Some(mir_opt_level) = self.config.rust_validate_mir_opts { - rustflags.arg("-Zvalidate-mir"); - rustflags.arg(&format!("-Zmir-opt-level={mir_opt_level}")); - } - // Always enable inlining MIR when building the standard library. - // Without this flag, MIR inlining is disabled when incremental compilation is enabled. - // That causes some mir-opt tests which inline functions from the standard library to - // break when incremental compilation is enabled. So this overrides the "no inlining - // during incremental builds" heuristic for the standard library. - rustflags.arg("-Zinline-mir"); - } - - // set rustc args passed from command line - let rustc_args = - self.config.cmd.rustc_args().iter().map(|s| s.to_string()).collect::>(); - if !rustc_args.is_empty() { - cargo.env("RUSTFLAGS", &rustc_args.join(" ")); - } - - Cargo { command: cargo, rustflags, rustdocflags, allow_features } - } - - /// Ensure that a given step is built, returning its output. This will - /// cache the step, so it is safe (and good!) to call this as often as - /// needed to ensure that all dependencies are built. - pub fn ensure(&'a self, step: S) -> S::Output { - { - let mut stack = self.stack.borrow_mut(); - for stack_step in stack.iter() { - // should skip - if stack_step.downcast_ref::().map_or(true, |stack_step| *stack_step != step) { - continue; - } - let mut out = String::new(); - out += &format!("\n\nCycle in build detected when adding {step:?}\n"); - for el in stack.iter().rev() { - out += &format!("\t{el:?}\n"); - } - panic!("{}", out); - } - if let Some(out) = self.cache.get(&step) { - self.verbose_than(1, &format!("{}c {:?}", " ".repeat(stack.len()), step)); - - return out; - } - self.verbose_than(1, &format!("{}> {:?}", " ".repeat(stack.len()), step)); - stack.push(Box::new(step.clone())); - } - - #[cfg(feature = "build-metrics")] - self.metrics.enter_step(&step, self); - - let (out, dur) = { - let start = Instant::now(); - let zero = Duration::new(0, 0); - let parent = self.time_spent_on_dependencies.replace(zero); - let out = step.clone().run(self); - let dur = start.elapsed(); - let deps = self.time_spent_on_dependencies.replace(parent + dur); - (out, dur - deps) - }; - - if self.config.print_step_timings && !self.config.dry_run() { - let step_string = format!("{step:?}"); - let brace_index = step_string.find("{").unwrap_or(0); - let type_string = type_name::(); - println!( - "[TIMING] {} {} -- {}.{:03}", - &type_string.strip_prefix("bootstrap::").unwrap_or(type_string), - &step_string[brace_index..], - dur.as_secs(), - dur.subsec_millis() - ); - } - - #[cfg(feature = "build-metrics")] - self.metrics.exit_step(self); - - { - let mut stack = self.stack.borrow_mut(); - let cur_step = stack.pop().expect("step stack empty"); - assert_eq!(cur_step.downcast_ref(), Some(&step)); - } - self.verbose_than(1, &format!("{}< {:?}", " ".repeat(self.stack.borrow().len()), step)); - self.cache.put(step, out.clone()); - out - } - - /// Ensure that a given step is built *only if it's supposed to be built by default*, returning - /// its output. This will cache the step, so it's safe (and good!) to call this as often as - /// needed to ensure that all dependencies are build. - pub(crate) fn ensure_if_default>>( - &'a self, - step: S, - kind: Kind, - ) -> S::Output { - let desc = StepDescription::from::(kind); - let should_run = (desc.should_run)(ShouldRun::new(self, desc.kind)); - - // Avoid running steps contained in --skip - for pathset in &should_run.paths { - if desc.is_excluded(self, pathset) { - return None; - } - } - - // Only execute if it's supposed to run as default - if desc.default && should_run.is_really_default() { self.ensure(step) } else { None } - } - - /// Checks if any of the "should_run" paths is in the `Builder` paths. - pub(crate) fn was_invoked_explicitly(&'a self, kind: Kind) -> bool { - let desc = StepDescription::from::(kind); - let should_run = (desc.should_run)(ShouldRun::new(self, desc.kind)); - - for path in &self.paths { - if should_run.paths.iter().any(|s| s.has(path, desc.kind)) - && !desc.is_excluded( - self, - &PathSet::Suite(TaskPath { path: path.clone(), kind: Some(desc.kind) }), - ) - { - return true; - } - } - - false - } - - pub(crate) fn maybe_open_in_browser(&self, path: impl AsRef) { - if self.was_invoked_explicitly::(Kind::Doc) { - self.open_in_browser(path); - } - } - - pub(crate) fn open_in_browser(&self, path: impl AsRef) { - if self.config.dry_run() || !self.config.cmd.open() { - return; - } - - let path = path.as_ref(); - self.info(&format!("Opening doc {}", path.display())); - if let Err(err) = opener::open(path) { - self.info(&format!("{err}\n")); - } - } -} - -#[cfg(test)] -mod tests; - -/// Represents flag values in `String` form with whitespace delimiter to pass it to the compiler later. -/// -/// `-Z crate-attr` flags will be applied recursively on the target code using the `rustc_parse::parser::Parser`. -/// See `rustc_builtin_macros::cmdline_attrs::inject` for more information. -#[derive(Debug, Clone)] -struct Rustflags(String, TargetSelection); - -impl Rustflags { - fn new(target: TargetSelection) -> Rustflags { - let mut ret = Rustflags(String::new(), target); - ret.propagate_cargo_env("RUSTFLAGS"); - ret - } - - /// By default, cargo will pick up on various variables in the environment. However, bootstrap - /// reuses those variables to pass additional flags to rustdoc, so by default they get overridden. - /// Explicitly add back any previous value in the environment. - /// - /// `prefix` is usually `RUSTFLAGS` or `RUSTDOCFLAGS`. - fn propagate_cargo_env(&mut self, prefix: &str) { - // Inherit `RUSTFLAGS` by default ... - self.env(prefix); - - // ... and also handle target-specific env RUSTFLAGS if they're configured. - let target_specific = format!("CARGO_TARGET_{}_{}", crate::envify(&self.1.triple), prefix); - self.env(&target_specific); - } - - fn env(&mut self, env: &str) { - if let Ok(s) = env::var(env) { - for part in s.split(' ') { - self.arg(part); - } - } - } - - fn arg(&mut self, arg: &str) -> &mut Self { - assert_eq!(arg.split(' ').count(), 1); - if !self.0.is_empty() { - self.0.push(' '); - } - self.0.push_str(arg); - self - } -} - -#[derive(Debug)] -pub struct Cargo { - command: Command, - rustflags: Rustflags, - rustdocflags: Rustflags, - allow_features: String, -} - -impl Cargo { - pub fn rustdocflag(&mut self, arg: &str) -> &mut Cargo { - self.rustdocflags.arg(arg); - self - } - pub fn rustflag(&mut self, arg: &str) -> &mut Cargo { - self.rustflags.arg(arg); - self - } - - pub fn arg(&mut self, arg: impl AsRef) -> &mut Cargo { - self.command.arg(arg.as_ref()); - self - } - - pub fn args(&mut self, args: I) -> &mut Cargo - where - I: IntoIterator, - S: AsRef, - { - for arg in args { - self.arg(arg.as_ref()); - } - self - } - - pub fn env(&mut self, key: impl AsRef, value: impl AsRef) -> &mut Cargo { - // These are managed through rustflag/rustdocflag interfaces. - assert_ne!(key.as_ref(), "RUSTFLAGS"); - assert_ne!(key.as_ref(), "RUSTDOCFLAGS"); - self.command.env(key.as_ref(), value.as_ref()); - self - } - - pub fn add_rustc_lib_path(&mut self, builder: &Builder<'_>, compiler: Compiler) { - builder.add_rustc_lib_path(compiler, &mut self.command); - } - - pub fn current_dir(&mut self, dir: &Path) -> &mut Cargo { - self.command.current_dir(dir); - self - } - - /// Adds nightly-only features that this invocation is allowed to use. - /// - /// By default, all nightly features are allowed. Once this is called, it - /// will be restricted to the given set. - pub fn allow_features(&mut self, features: &str) -> &mut Cargo { - if !self.allow_features.is_empty() { - self.allow_features.push(','); - } - self.allow_features.push_str(features); - self - } -} - -impl From for Command { - fn from(mut cargo: Cargo) -> Command { - let rustflags = &cargo.rustflags.0; - if !rustflags.is_empty() { - cargo.command.env("RUSTFLAGS", rustflags); - } - - let rustdocflags = &cargo.rustdocflags.0; - if !rustdocflags.is_empty() { - cargo.command.env("RUSTDOCFLAGS", rustdocflags); - } - - if !cargo.allow_features.is_empty() { - cargo.command.env("RUSTC_ALLOW_FEATURES", cargo.allow_features); - } - - cargo.command - } -} diff --git a/src/bootstrap/builder/tests.rs b/src/bootstrap/builder/tests.rs deleted file mode 100644 index 80e66622e..000000000 --- a/src/bootstrap/builder/tests.rs +++ /dev/null @@ -1,702 +0,0 @@ -use super::*; -use crate::config::{Config, DryRun, TargetSelection}; -use crate::doc::DocumentationFormat; -use std::thread; - -fn configure(cmd: &str, host: &[&str], target: &[&str]) -> Config { - configure_with_args(&[cmd.to_owned()], host, target) -} - -fn configure_with_args(cmd: &[String], host: &[&str], target: &[&str]) -> Config { - let mut config = Config::parse(cmd); - // don't save toolstates - config.save_toolstates = None; - config.dry_run = DryRun::SelfCheck; - - // Ignore most submodules, since we don't need them for a dry run. - // But make sure to check out the `doc` and `rust-analyzer` submodules, since some steps need them - // just to know which commands to run. - let submodule_build = Build::new(Config { - // don't include LLVM, so CI doesn't require ninja/cmake to be installed - rust_codegen_backends: vec![], - ..Config::parse(&["check".to_owned()]) - }); - submodule_build.update_submodule(Path::new("src/doc/book")); - submodule_build.update_submodule(Path::new("src/tools/rust-analyzer")); - config.submodules = Some(false); - - config.ninja_in_file = false; - // try to avoid spurious failures in dist where we create/delete each others file - // HACK: rather than pull in `tempdir`, use the one that cargo has conveniently created for us - let dir = Path::new(env!("OUT_DIR")) - .join("tmp-rustbuild-tests") - .join(&thread::current().name().unwrap_or("unknown").replace(":", "-")); - t!(fs::create_dir_all(&dir)); - config.out = dir; - config.build = TargetSelection::from_user("A"); - config.hosts = host.iter().map(|s| TargetSelection::from_user(s)).collect(); - config.targets = target.iter().map(|s| TargetSelection::from_user(s)).collect(); - config -} - -fn first(v: Vec<(A, B)>) -> Vec { - v.into_iter().map(|(a, _)| a).collect::>() -} - -fn run_build(paths: &[PathBuf], config: Config) -> Cache { - let kind = config.cmd.kind(); - let build = Build::new(config); - let builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(kind), paths); - builder.cache -} - -fn check_cli(paths: [&str; N]) { - run_build( - &paths.map(PathBuf::from), - configure_with_args(&paths.map(String::from), &["A"], &["A"]), - ); -} - -macro_rules! std { - ($host:ident => $target:ident, stage = $stage:literal) => { - compile::Std::new( - Compiler { host: TargetSelection::from_user(stringify!($host)), stage: $stage }, - TargetSelection::from_user(stringify!($target)), - ) - }; -} - -macro_rules! doc_std { - ($host:ident => $target:ident, stage = $stage:literal) => {{ - let config = configure("doc", &["A"], &["A"]); - let build = Build::new(config); - let builder = Builder::new(&build); - doc::Std::new( - $stage, - TargetSelection::from_user(stringify!($target)), - &builder, - DocumentationFormat::HTML, - ) - }}; -} - -macro_rules! rustc { - ($host:ident => $target:ident, stage = $stage:literal) => { - compile::Rustc::new( - Compiler { host: TargetSelection::from_user(stringify!($host)), stage: $stage }, - TargetSelection::from_user(stringify!($target)), - ) - }; -} - -#[test] -fn test_valid() { - // make sure multi suite paths are accepted - check_cli(["test", "tests/ui/attr-start.rs", "tests/ui/attr-shebang.rs"]); -} - -#[test] -#[should_panic] -fn test_invalid() { - // make sure that invalid paths are caught, even when combined with valid paths - check_cli(["test", "library/std", "x"]); -} - -#[test] -fn test_intersection() { - let set = |paths: &[&str]| { - PathSet::Set(paths.into_iter().map(|p| TaskPath { path: p.into(), kind: None }).collect()) - }; - let library_set = set(&["library/core", "library/alloc", "library/std"]); - let mut command_paths = - vec![Path::new("library/core"), Path::new("library/alloc"), Path::new("library/stdarch")]; - let subset = library_set.intersection_removing_matches(&mut command_paths, Kind::Build); - assert_eq!(subset, set(&["library/core", "library/alloc"]),); - assert_eq!(command_paths, vec![Path::new("library/stdarch")]); -} - -#[test] -fn test_exclude() { - let mut config = configure("test", &["A"], &["A"]); - config.skip = vec!["src/tools/tidy".into()]; - let cache = run_build(&[], config); - - // Ensure we have really excluded tidy - assert!(!cache.contains::()); - - // Ensure other tests are not affected. - assert!(cache.contains::()); -} - -#[test] -fn test_exclude_kind() { - let path = PathBuf::from("compiler/rustc_data_structures"); - - let mut config = configure("test", &["A"], &["A"]); - // Ensure our test is valid, and `test::Rustc` would be run without the exclude. - assert!(run_build(&[], config.clone()).contains::()); - // Ensure tests for rustc are not skipped. - config.skip = vec![path.clone()]; - assert!(run_build(&[], config.clone()).contains::()); - // Ensure builds for rustc are not skipped. - assert!(run_build(&[], config).contains::()); -} - -/// Ensure that if someone passes both a single crate and `library`, all library crates get built. -#[test] -fn alias_and_path_for_library() { - let mut cache = - run_build(&["library".into(), "core".into()], configure("build", &["A"], &["A"])); - assert_eq!( - first(cache.all::()), - &[std!(A => A, stage = 0), std!(A => A, stage = 1)] - ); - - let mut cache = run_build(&["library".into(), "core".into()], configure("doc", &["A"], &["A"])); - assert_eq!(first(cache.all::()), &[doc_std!(A => A, stage = 0)]); -} - -#[test] -fn test_beta_rev_parsing() { - use crate::extract_beta_rev; - - // single digit revision - assert_eq!(extract_beta_rev("1.99.9-beta.7 (xxxxxx)"), Some("7".to_string())); - // multiple digits - assert_eq!(extract_beta_rev("1.99.9-beta.777 (xxxxxx)"), Some("777".to_string())); - // nightly channel (no beta revision) - assert_eq!(extract_beta_rev("1.99.9-nightly (xxxxxx)"), None); - // stable channel (no beta revision) - assert_eq!(extract_beta_rev("1.99.9 (xxxxxxx)"), None); - // invalid string - assert_eq!(extract_beta_rev("invalid"), None); -} - -mod defaults { - use super::{configure, first, run_build}; - use crate::builder::*; - use crate::Config; - use pretty_assertions::assert_eq; - - #[test] - fn build_default() { - let mut cache = run_build(&[], configure("build", &["A"], &["A"])); - - let a = TargetSelection::from_user("A"); - assert_eq!( - first(cache.all::()), - &[std!(A => A, stage = 0), std!(A => A, stage = 1),] - ); - assert!(!cache.all::().is_empty()); - // Make sure rustdoc is only built once. - assert_eq!( - first(cache.all::()), - // Recall that rustdoc stages are off-by-one - // - this is the compiler it's _linked_ to, not built with. - &[tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }], - ); - assert_eq!(first(cache.all::()), &[rustc!(A => A, stage = 0)],); - } - - #[test] - fn build_stage_0() { - let config = Config { stage: 0, ..configure("build", &["A"], &["A"]) }; - let mut cache = run_build(&[], config); - - let a = TargetSelection::from_user("A"); - assert_eq!(first(cache.all::()), &[std!(A => A, stage = 0)]); - assert!(!cache.all::().is_empty()); - assert_eq!( - first(cache.all::()), - // This is the beta rustdoc. - // Add an assert here to make sure this is the only rustdoc built. - &[tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } }], - ); - assert!(cache.all::().is_empty()); - } - - #[test] - fn build_cross_compile() { - let config = Config { stage: 1, ..configure("build", &["A", "B"], &["A", "B"]) }; - let mut cache = run_build(&[], config); - - let a = TargetSelection::from_user("A"); - let b = TargetSelection::from_user("B"); - - // Ideally, this build wouldn't actually have `target: a` - // rustdoc/rustcc/std here (the user only requested a host=B build, so - // there's not really a need for us to build for target A in this case - // (since we're producing stage 1 libraries/binaries). But currently - // rustbuild is just a bit buggy here; this should be fixed though. - assert_eq!( - first(cache.all::()), - &[ - std!(A => A, stage = 0), - std!(A => A, stage = 1), - std!(A => B, stage = 0), - std!(A => B, stage = 1), - ] - ); - assert_eq!( - first(cache.all::()), - &[ - compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, - compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, - compile::Assemble { target_compiler: Compiler { host: b, stage: 1 } }, - ] - ); - assert_eq!( - first(cache.all::()), - &[ - tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }, - tool::Rustdoc { compiler: Compiler { host: b, stage: 1 } }, - ], - ); - assert_eq!( - first(cache.all::()), - &[rustc!(A => A, stage = 0), rustc!(A => B, stage = 0),] - ); - } - - #[test] - fn doc_default() { - let mut config = configure("doc", &["A"], &["A"]); - config.compiler_docs = true; - config.cmd = Subcommand::Doc { open: false, json: false }; - let mut cache = run_build(&[], config); - let a = TargetSelection::from_user("A"); - - // error_index_generator uses stage 0 to share rustdoc artifacts with the - // rustdoc tool. - assert_eq!(first(cache.all::()), &[doc::ErrorIndex { target: a },]); - assert_eq!( - first(cache.all::()), - &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 0 } }] - ); - // docs should be built with the beta compiler, not with the stage0 artifacts. - // recall that rustdoc is off-by-one: `stage` is the compiler rustdoc is _linked_ to, - // not the one it was built by. - assert_eq!( - first(cache.all::()), - &[tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } },] - ); - } -} - -mod dist { - use super::{first, run_build, Config}; - use crate::builder::*; - use pretty_assertions::assert_eq; - - fn configure(host: &[&str], target: &[&str]) -> Config { - Config { stage: 2, ..super::configure("dist", host, target) } - } - - #[test] - fn dist_baseline() { - let mut cache = run_build(&[], configure(&["A"], &["A"])); - - let a = TargetSelection::from_user("A"); - - assert_eq!(first(cache.all::()), &[dist::Docs { host: a },]); - assert_eq!(first(cache.all::()), &[dist::Mingw { host: a },]); - assert_eq!( - first(cache.all::()), - &[dist::Rustc { compiler: Compiler { host: a, stage: 2 } },] - ); - assert_eq!( - first(cache.all::()), - &[dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a },] - ); - assert_eq!(first(cache.all::()), &[dist::Src]); - // Make sure rustdoc is only built once. - assert_eq!( - first(cache.all::()), - &[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },] - ); - } - - #[test] - fn dist_with_targets() { - let mut cache = run_build(&[], configure(&["A"], &["A", "B"])); - - let a = TargetSelection::from_user("A"); - let b = TargetSelection::from_user("B"); - - assert_eq!( - first(cache.all::()), - &[dist::Docs { host: a }, dist::Docs { host: b },] - ); - assert_eq!( - first(cache.all::()), - &[dist::Mingw { host: a }, dist::Mingw { host: b },] - ); - assert_eq!( - first(cache.all::()), - &[dist::Rustc { compiler: Compiler { host: a, stage: 2 } },] - ); - assert_eq!( - first(cache.all::()), - &[ - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, - dist::Std { compiler: Compiler { host: a, stage: 2 }, target: b }, - ] - ); - assert_eq!(first(cache.all::()), &[dist::Src]); - } - - #[test] - fn dist_with_hosts() { - let mut cache = run_build(&[], configure(&["A", "B"], &["A", "B"])); - - let a = TargetSelection::from_user("A"); - let b = TargetSelection::from_user("B"); - - assert_eq!( - first(cache.all::()), - &[dist::Docs { host: a }, dist::Docs { host: b },] - ); - assert_eq!( - first(cache.all::()), - &[dist::Mingw { host: a }, dist::Mingw { host: b },] - ); - assert_eq!( - first(cache.all::()), - &[ - dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, - dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, - ] - ); - assert_eq!( - first(cache.all::()), - &[ - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, - ] - ); - assert_eq!( - first(cache.all::()), - &[ - std!(A => A, stage = 0), - std!(A => A, stage = 1), - std!(A => A, stage = 2), - std!(A => B, stage = 1), - std!(A => B, stage = 2), - ], - ); - assert_eq!(first(cache.all::()), &[dist::Src]); - } - - #[test] - fn dist_only_cross_host() { - let b = TargetSelection::from_user("B"); - let mut config = configure(&["A", "B"], &["A", "B"]); - config.docs = false; - config.extended = true; - config.hosts = vec![b]; - let mut cache = run_build(&[], config); - - assert_eq!( - first(cache.all::()), - &[dist::Rustc { compiler: Compiler { host: b, stage: 2 } },] - ); - assert_eq!( - first(cache.all::()), - &[rustc!(A => A, stage = 0), rustc!(A => B, stage = 1),] - ); - } - - #[test] - fn dist_with_targets_and_hosts() { - let mut cache = run_build(&[], configure(&["A", "B"], &["A", "B", "C"])); - - let a = TargetSelection::from_user("A"); - let b = TargetSelection::from_user("B"); - let c = TargetSelection::from_user("C"); - - assert_eq!( - first(cache.all::()), - &[dist::Docs { host: a }, dist::Docs { host: b }, dist::Docs { host: c },] - ); - assert_eq!( - first(cache.all::()), - &[dist::Mingw { host: a }, dist::Mingw { host: b }, dist::Mingw { host: c },] - ); - assert_eq!( - first(cache.all::()), - &[ - dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, - dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, - ] - ); - assert_eq!( - first(cache.all::()), - &[ - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, - dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c }, - ] - ); - assert_eq!(first(cache.all::()), &[dist::Src]); - } - - #[test] - fn dist_with_empty_host() { - let config = configure(&[], &["C"]); - let mut cache = run_build(&[], config); - - let a = TargetSelection::from_user("A"); - let c = TargetSelection::from_user("C"); - - assert_eq!(first(cache.all::()), &[dist::Docs { host: c },]); - assert_eq!(first(cache.all::()), &[dist::Mingw { host: c },]); - assert_eq!( - first(cache.all::()), - &[dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c },] - ); - } - - #[test] - fn dist_with_same_targets_and_hosts() { - let mut cache = run_build(&[], configure(&["A", "B"], &["A", "B"])); - - let a = TargetSelection::from_user("A"); - let b = TargetSelection::from_user("B"); - - assert_eq!( - first(cache.all::()), - &[dist::Docs { host: a }, dist::Docs { host: b },] - ); - assert_eq!( - first(cache.all::()), - &[dist::Mingw { host: a }, dist::Mingw { host: b },] - ); - assert_eq!( - first(cache.all::()), - &[ - dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, - dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, - ] - ); - assert_eq!( - first(cache.all::()), - &[ - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, - ] - ); - assert_eq!(first(cache.all::()), &[dist::Src]); - assert_eq!( - first(cache.all::()), - &[ - std!(A => A, stage = 0), - std!(A => A, stage = 1), - std!(A => A, stage = 2), - std!(A => B, stage = 1), - std!(A => B, stage = 2), - ] - ); - assert_eq!( - first(cache.all::()), - &[ - compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, - compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, - compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } }, - compile::Assemble { target_compiler: Compiler { host: b, stage: 2 } }, - ] - ); - } - - #[test] - fn build_all() { - let build = Build::new(configure(&["A", "B"], &["A", "B", "C"])); - let mut builder = Builder::new(&build); - builder.run_step_descriptions( - &Builder::get_step_descriptions(Kind::Build), - &["compiler/rustc".into(), "library".into()], - ); - - assert_eq!( - first(builder.cache.all::()), - &[ - std!(A => A, stage = 0), - std!(A => A, stage = 1), - std!(A => A, stage = 2), - std!(A => B, stage = 1), - std!(A => B, stage = 2), - std!(A => C, stage = 2), - ] - ); - assert_eq!(builder.cache.all::().len(), 5); - assert_eq!( - first(builder.cache.all::()), - &[ - rustc!(A => A, stage = 0), - rustc!(A => A, stage = 1), - rustc!(A => A, stage = 2), - rustc!(A => B, stage = 1), - rustc!(A => B, stage = 2), - ] - ); - } - - #[test] - fn build_with_empty_host() { - let config = configure(&[], &["C"]); - let build = Build::new(config); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]); - - let a = TargetSelection::from_user("A"); - - assert_eq!( - first(builder.cache.all::()), - &[std!(A => A, stage = 0), std!(A => A, stage = 1), std!(A => C, stage = 2),] - ); - assert_eq!( - first(builder.cache.all::()), - &[ - compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, - compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, - compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } }, - ] - ); - assert_eq!( - first(builder.cache.all::()), - &[rustc!(A => A, stage = 0), rustc!(A => A, stage = 1),] - ); - } - - #[test] - fn test_with_no_doc_stage0() { - let mut config = configure(&["A"], &["A"]); - config.stage = 0; - config.paths = vec!["library/std".into()]; - config.cmd = Subcommand::Test { - test_args: vec![], - rustc_args: vec![], - no_fail_fast: false, - no_doc: true, - doc: false, - bless: false, - force_rerun: false, - compare_mode: None, - rustfix_coverage: false, - pass: None, - run: None, - only_modified: false, - skip: vec![], - extra_checks: None, - }; - - let build = Build::new(config); - let mut builder = Builder::new(&build); - - let host = TargetSelection::from_user("A"); - - builder.run_step_descriptions( - &[StepDescription::from::(Kind::Test)], - &["library/std".into()], - ); - - // Ensure we don't build any compiler artifacts. - assert!(!builder.cache.contains::()); - assert_eq!( - first(builder.cache.all::()), - &[test::Crate { - compiler: Compiler { host, stage: 0 }, - target: host, - mode: Mode::Std, - crates: vec![INTERNER.intern_str("std")], - },] - ); - } - - #[test] - fn doc_ci() { - let mut config = configure(&["A"], &["A"]); - config.compiler_docs = true; - config.cmd = Subcommand::Doc { open: false, json: false }; - let build = Build::new(config); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Doc), &[]); - let a = TargetSelection::from_user("A"); - - // error_index_generator uses stage 1 to share rustdoc artifacts with the - // rustdoc tool. - assert_eq!( - first(builder.cache.all::()), - &[doc::ErrorIndex { target: a },] - ); - assert_eq!( - first(builder.cache.all::()), - &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 1 } }] - ); - // This is actually stage 1, but Rustdoc::run swaps out the compiler with - // stage minus 1 if --stage is not 0. Very confusing! - assert_eq!( - first(builder.cache.all::()), - &[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },] - ); - } - - #[test] - fn test_docs() { - // Behavior of `x.py test` doing various documentation tests. - let mut config = configure(&["A"], &["A"]); - config.cmd = Subcommand::Test { - test_args: vec![], - rustc_args: vec![], - no_fail_fast: false, - doc: true, - no_doc: false, - skip: vec![], - bless: false, - force_rerun: false, - compare_mode: None, - rustfix_coverage: false, - pass: None, - run: None, - only_modified: false, - extra_checks: None, - }; - // Make sure rustfmt binary not being found isn't an error. - config.channel = "beta".to_string(); - let build = Build::new(config); - let mut builder = Builder::new(&build); - - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Test), &[]); - let a = TargetSelection::from_user("A"); - - // error_index_generator uses stage 1 to share rustdoc artifacts with the - // rustdoc tool. - assert_eq!( - first(builder.cache.all::()), - &[doc::ErrorIndex { target: a },] - ); - assert_eq!( - first(builder.cache.all::()), - &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 1 } }] - ); - // Unfortunately rustdoc is built twice. Once from stage1 for compiletest - // (and other things), and once from stage0 for std crates. Ideally it - // would only be built once. If someone wants to fix this, it might be - // worth investigating if it would be possible to test std from stage1. - // Note that the stages here are +1 than what they actually are because - // Rustdoc::run swaps out the compiler with stage minus 1 if --stage is - // not 0. - // - // The stage 0 copy is the one downloaded for bootstrapping. It is - // (currently) needed to run "cargo test" on the linkchecker, and - // should be relatively "free". - assert_eq!( - first(builder.cache.all::()), - &[ - tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } }, - tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }, - tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } }, - ] - ); - } -} diff --git a/src/bootstrap/cache.rs b/src/bootstrap/cache.rs deleted file mode 100644 index 53e4ff034..000000000 --- a/src/bootstrap/cache.rs +++ /dev/null @@ -1,272 +0,0 @@ -use std::any::{Any, TypeId}; -use std::borrow::Borrow; -use std::cell::RefCell; -use std::cmp::Ordering; -use std::collections::HashMap; -use std::fmt; -use std::hash::{Hash, Hasher}; -use std::marker::PhantomData; -use std::mem; -use std::ops::Deref; -use std::path::PathBuf; -use std::sync::Mutex; - -// FIXME: replace with std::lazy after it gets stabilized and reaches beta -use once_cell::sync::Lazy; - -use crate::builder::Step; - -pub struct Interned(usize, PhantomData<*const T>); - -impl Default for Interned { - fn default() -> Self { - T::default().intern() - } -} - -impl Copy for Interned {} -impl Clone for Interned { - fn clone(&self) -> Interned { - *self - } -} - -impl PartialEq for Interned { - fn eq(&self, other: &Self) -> bool { - self.0 == other.0 - } -} -impl Eq for Interned {} - -impl PartialEq for Interned { - fn eq(&self, other: &str) -> bool { - *self == other - } -} -impl<'a> PartialEq<&'a str> for Interned { - fn eq(&self, other: &&str) -> bool { - **self == **other - } -} -impl<'a, T> PartialEq<&'a Interned> for Interned { - fn eq(&self, other: &&Self) -> bool { - self.0 == other.0 - } -} -impl<'a, T> PartialEq> for &'a Interned { - fn eq(&self, other: &Interned) -> bool { - self.0 == other.0 - } -} - -unsafe impl Send for Interned {} -unsafe impl Sync for Interned {} - -impl fmt::Display for Interned { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let s: &str = &*self; - f.write_str(s) - } -} - -impl fmt::Debug for Interned -where - Self: Deref, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let s: &U = &*self; - f.write_fmt(format_args!("{s:?}")) - } -} - -impl Hash for Interned { - fn hash(&self, state: &mut H) { - let l = T::intern_cache().lock().unwrap(); - l.get(*self).hash(state) - } -} - -impl Deref for Interned { - type Target = T::Target; - fn deref(&self) -> &Self::Target { - let l = T::intern_cache().lock().unwrap(); - unsafe { mem::transmute::<&Self::Target, &Self::Target>(l.get(*self)) } - } -} - -impl, U: ?Sized> AsRef for Interned { - fn as_ref(&self) -> &U { - let l = T::intern_cache().lock().unwrap(); - unsafe { mem::transmute::<&U, &U>(l.get(*self).as_ref()) } - } -} - -impl PartialOrd for Interned { - fn partial_cmp(&self, other: &Self) -> Option { - let l = T::intern_cache().lock().unwrap(); - l.get(*self).partial_cmp(l.get(*other)) - } -} - -impl Ord for Interned { - fn cmp(&self, other: &Self) -> Ordering { - let l = T::intern_cache().lock().unwrap(); - l.get(*self).cmp(l.get(*other)) - } -} - -struct TyIntern { - items: Vec, - set: HashMap>, -} - -impl Default for TyIntern { - fn default() -> Self { - TyIntern { items: Vec::new(), set: Default::default() } - } -} - -impl TyIntern { - fn intern_borrow(&mut self, item: &B) -> Interned - where - B: Eq + Hash + ToOwned + ?Sized, - T: Borrow, - { - if let Some(i) = self.set.get(&item) { - return *i; - } - let item = item.to_owned(); - let interned = Interned(self.items.len(), PhantomData::<*const T>); - self.set.insert(item.clone(), interned); - self.items.push(item); - interned - } - - fn intern(&mut self, item: T) -> Interned { - if let Some(i) = self.set.get(&item) { - return *i; - } - let interned = Interned(self.items.len(), PhantomData::<*const T>); - self.set.insert(item.clone(), interned); - self.items.push(item); - interned - } - - fn get(&self, i: Interned) -> &T { - &self.items[i.0] - } -} - -#[derive(Default)] -pub struct Interner { - strs: Mutex>, - paths: Mutex>, - lists: Mutex>>, -} - -trait Internable: Clone + Eq + Hash + 'static { - fn intern_cache() -> &'static Mutex>; - - fn intern(self) -> Interned { - Self::intern_cache().lock().unwrap().intern(self) - } -} - -impl Internable for String { - fn intern_cache() -> &'static Mutex> { - &INTERNER.strs - } -} - -impl Internable for PathBuf { - fn intern_cache() -> &'static Mutex> { - &INTERNER.paths - } -} - -impl Internable for Vec { - fn intern_cache() -> &'static Mutex> { - &INTERNER.lists - } -} - -impl Interner { - pub fn intern_str(&self, s: &str) -> Interned { - self.strs.lock().unwrap().intern_borrow(s) - } - pub fn intern_string(&self, s: String) -> Interned { - self.strs.lock().unwrap().intern(s) - } - - pub fn intern_path(&self, s: PathBuf) -> Interned { - self.paths.lock().unwrap().intern(s) - } - - pub fn intern_list(&self, v: Vec) -> Interned> { - self.lists.lock().unwrap().intern(v) - } -} - -pub static INTERNER: Lazy = Lazy::new(Interner::default); - -/// This is essentially a `HashMap` which allows storing any type in its input and -/// any type in its output. It is a write-once cache; values are never evicted, -/// which means that references to the value can safely be returned from the -/// `get()` method. -#[derive(Debug)] -pub struct Cache( - RefCell< - HashMap< - TypeId, - Box, // actually a HashMap> - >, - >, -); - -impl Cache { - pub fn new() -> Cache { - Cache(RefCell::new(HashMap::new())) - } - - pub fn put(&self, step: S, value: S::Output) { - let mut cache = self.0.borrow_mut(); - let type_id = TypeId::of::(); - let stepcache = cache - .entry(type_id) - .or_insert_with(|| Box::new(HashMap::::new())) - .downcast_mut::>() - .expect("invalid type mapped"); - assert!(!stepcache.contains_key(&step), "processing {step:?} a second time"); - stepcache.insert(step, value); - } - - pub fn get(&self, step: &S) -> Option { - let mut cache = self.0.borrow_mut(); - let type_id = TypeId::of::(); - let stepcache = cache - .entry(type_id) - .or_insert_with(|| Box::new(HashMap::::new())) - .downcast_mut::>() - .expect("invalid type mapped"); - stepcache.get(step).cloned() - } -} - -#[cfg(test)] -impl Cache { - pub fn all(&mut self) -> Vec<(S, S::Output)> { - let cache = self.0.get_mut(); - let type_id = TypeId::of::(); - let mut v = cache - .remove(&type_id) - .map(|b| b.downcast::>().expect("correct type")) - .map(|m| m.into_iter().collect::>()) - .unwrap_or_default(); - v.sort_by_key(|(s, _)| s.clone()); - v - } - - pub fn contains(&self) -> bool { - self.0.borrow().contains_key(&TypeId::of::()) - } -} diff --git a/src/bootstrap/cc_detect.rs b/src/bootstrap/cc_detect.rs deleted file mode 100644 index 2496c2a9d..000000000 --- a/src/bootstrap/cc_detect.rs +++ /dev/null @@ -1,273 +0,0 @@ -//! C-compiler probing and detection. -//! -//! This module will fill out the `cc` and `cxx` maps of `Build` by looking for -//! C and C++ compilers for each target configured. A compiler is found through -//! a number of vectors (in order of precedence) -//! -//! 1. Configuration via `target.$target.cc` in `config.toml`. -//! 2. Configuration via `target.$target.android-ndk` in `config.toml`, if -//! applicable -//! 3. Special logic to probe on OpenBSD -//! 4. The `CC_$target` environment variable. -//! 5. The `CC` environment variable. -//! 6. "cc" -//! -//! Some of this logic is implemented here, but much of it is farmed out to the -//! `cc` crate itself, so we end up having the same fallbacks as there. -//! Similar logic is then used to find a C++ compiler, just some s/cc/c++/ is -//! used. -//! -//! It is intended that after this module has run no C/C++ compiler will -//! ever be probed for. Instead the compilers found here will be used for -//! everything. - -use std::collections::HashSet; -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::{env, iter}; - -use crate::config::{Target, TargetSelection}; -use crate::util::output; -use crate::{Build, CLang, GitRepo}; - -// The `cc` crate doesn't provide a way to obtain a path to the detected archiver, -// so use some simplified logic here. First we respect the environment variable `AR`, then -// try to infer the archiver path from the C compiler path. -// In the future this logic should be replaced by calling into the `cc` crate. -fn cc2ar(cc: &Path, target: TargetSelection) -> Option { - if let Some(ar) = env::var_os(format!("AR_{}", target.triple.replace("-", "_"))) { - Some(PathBuf::from(ar)) - } else if let Some(ar) = env::var_os("AR") { - Some(PathBuf::from(ar)) - } else if target.contains("msvc") { - None - } else if target.contains("musl") { - Some(PathBuf::from("ar")) - } else if target.contains("openbsd") { - Some(PathBuf::from("ar")) - } else if target.contains("vxworks") { - Some(PathBuf::from("wr-ar")) - } else if target.contains("android") { - Some(cc.parent().unwrap().join(PathBuf::from("llvm-ar"))) - } else { - let parent = cc.parent().unwrap(); - let file = cc.file_name().unwrap().to_str().unwrap(); - for suffix in &["gcc", "cc", "clang"] { - if let Some(idx) = file.rfind(suffix) { - let mut file = file[..idx].to_owned(); - file.push_str("ar"); - return Some(parent.join(&file)); - } - } - Some(parent.join(file)) - } -} - -fn new_cc_build(build: &Build, target: TargetSelection) -> cc::Build { - let mut cfg = cc::Build::new(); - cfg.cargo_metadata(false) - .opt_level(2) - .warnings(false) - .debug(false) - // Compress debuginfo - .flag_if_supported("-gz") - .target(&target.triple) - .host(&build.build.triple); - match build.crt_static(target) { - Some(a) => { - cfg.static_crt(a); - } - None => { - if target.contains("msvc") { - cfg.static_crt(true); - } - if target.contains("musl") { - cfg.static_flag(true); - } - } - } - cfg -} - -pub fn find(build: &Build) { - // For all targets we're going to need a C compiler for building some shims - // and such as well as for being a linker for Rust code. - let targets = build - .targets - .iter() - .chain(&build.hosts) - .cloned() - .chain(iter::once(build.build)) - .collect::>(); - for target in targets.into_iter() { - find_target(build, target); - } -} - -pub fn find_target(build: &Build, target: TargetSelection) { - let mut cfg = new_cc_build(build, target); - let config = build.config.target_config.get(&target); - if let Some(cc) = config.and_then(|c| c.cc.as_ref()) { - cfg.compiler(cc); - } else { - set_compiler(&mut cfg, Language::C, target, config, build); - } - - let compiler = cfg.get_compiler(); - let ar = if let ar @ Some(..) = config.and_then(|c| c.ar.clone()) { - ar - } else { - cc2ar(compiler.path(), target) - }; - - build.cc.borrow_mut().insert(target, compiler.clone()); - let cflags = build.cflags(target, GitRepo::Rustc, CLang::C); - - // If we use llvm-libunwind, we will need a C++ compiler as well for all targets - // We'll need one anyways if the target triple is also a host triple - let mut cfg = new_cc_build(build, target); - cfg.cpp(true); - let cxx_configured = if let Some(cxx) = config.and_then(|c| c.cxx.as_ref()) { - cfg.compiler(cxx); - true - } else if build.hosts.contains(&target) || build.build == target { - set_compiler(&mut cfg, Language::CPlusPlus, target, config, build); - true - } else { - // Use an auto-detected compiler (or one configured via `CXX_target_triple` env vars). - cfg.try_get_compiler().is_ok() - }; - - // for VxWorks, record CXX compiler which will be used in lib.rs:linker() - if cxx_configured || target.contains("vxworks") { - let compiler = cfg.get_compiler(); - build.cxx.borrow_mut().insert(target, compiler); - } - - build.verbose(&format!("CC_{} = {:?}", &target.triple, build.cc(target))); - build.verbose(&format!("CFLAGS_{} = {:?}", &target.triple, cflags)); - if let Ok(cxx) = build.cxx(target) { - let cxxflags = build.cflags(target, GitRepo::Rustc, CLang::Cxx); - build.verbose(&format!("CXX_{} = {:?}", &target.triple, cxx)); - build.verbose(&format!("CXXFLAGS_{} = {:?}", &target.triple, cxxflags)); - } - if let Some(ar) = ar { - build.verbose(&format!("AR_{} = {:?}", &target.triple, ar)); - build.ar.borrow_mut().insert(target, ar); - } - - if let Some(ranlib) = config.and_then(|c| c.ranlib.clone()) { - build.ranlib.borrow_mut().insert(target, ranlib); - } -} - -fn set_compiler( - cfg: &mut cc::Build, - compiler: Language, - target: TargetSelection, - config: Option<&Target>, - build: &Build, -) { - match &*target.triple { - // When compiling for android we may have the NDK configured in the - // config.toml in which case we look there. Otherwise the default - // compiler already takes into account the triple in question. - t if t.contains("android") => { - if let Some(ndk) = config.and_then(|c| c.ndk.as_ref()) { - cfg.compiler(ndk_compiler(compiler, &*target.triple, ndk)); - } - } - - // The default gcc version from OpenBSD may be too old, try using egcc, - // which is a gcc version from ports, if this is the case. - t if t.contains("openbsd") => { - let c = cfg.get_compiler(); - let gnu_compiler = compiler.gcc(); - if !c.path().ends_with(gnu_compiler) { - return; - } - - let output = output(c.to_command().arg("--version")); - let i = match output.find(" 4.") { - Some(i) => i, - None => return, - }; - match output[i + 3..].chars().next().unwrap() { - '0'..='6' => {} - _ => return, - } - let alternative = format!("e{gnu_compiler}"); - if Command::new(&alternative).output().is_ok() { - cfg.compiler(alternative); - } - } - - "mips-unknown-linux-musl" => { - if cfg.get_compiler().path().to_str() == Some("gcc") { - cfg.compiler("mips-linux-musl-gcc"); - } - } - "mipsel-unknown-linux-musl" => { - if cfg.get_compiler().path().to_str() == Some("gcc") { - cfg.compiler("mipsel-linux-musl-gcc"); - } - } - - t if t.contains("musl") => { - if let Some(root) = build.musl_root(target) { - let guess = root.join("bin/musl-gcc"); - if guess.exists() { - cfg.compiler(guess); - } - } - } - - _ => {} - } -} - -pub(crate) fn ndk_compiler(compiler: Language, triple: &str, ndk: &Path) -> PathBuf { - let mut triple_iter = triple.split("-"); - let triple_translated = if let Some(arch) = triple_iter.next() { - let arch_new = match arch { - "arm" | "armv7" | "armv7neon" | "thumbv7" | "thumbv7neon" => "armv7a", - other => other, - }; - std::iter::once(arch_new).chain(triple_iter).collect::>().join("-") - } else { - triple.to_string() - }; - - // API 19 is the earliest API level supported by NDK r25b but AArch64 and x86_64 support - // begins at API level 21. - let api_level = - if triple.contains("aarch64") || triple.contains("x86_64") { "21" } else { "19" }; - let compiler = format!("{}{}-{}", triple_translated, api_level, compiler.clang()); - ndk.join("bin").join(compiler) -} - -/// The target programming language for a native compiler. -pub(crate) enum Language { - /// The compiler is targeting C. - C, - /// The compiler is targeting C++. - CPlusPlus, -} - -impl Language { - /// Obtains the name of a compiler in the GCC collection. - fn gcc(self) -> &'static str { - match self { - Language::C => "gcc", - Language::CPlusPlus => "g++", - } - } - - /// Obtains the name of a compiler in the clang suite. - fn clang(self) -> &'static str { - match self { - Language::C => "clang", - Language::CPlusPlus => "clang++", - } - } -} diff --git a/src/bootstrap/channel.rs b/src/bootstrap/channel.rs deleted file mode 100644 index 870185740..000000000 --- a/src/bootstrap/channel.rs +++ /dev/null @@ -1,160 +0,0 @@ -//! Build configuration for Rust's release channels. -//! -//! Implements the stable/beta/nightly channel distinctions by setting various -//! flags like the `unstable_features`, calculating variables like `release` and -//! `package_vers`, and otherwise indicating to the compiler what it should -//! print out as part of its version information. - -use std::fs; -use std::path::Path; -use std::process::Command; - -use crate::util::output; -use crate::util::t; -use crate::Build; - -#[derive(Clone, Default)] -pub enum GitInfo { - /// This is not a git repository. - #[default] - Absent, - /// This is a git repository. - /// If the info should be used (`omit_git_hash` is false), this will be - /// `Some`, otherwise it will be `None`. - Present(Option), - /// This is not a git repository, but the info can be fetched from the - /// `git-commit-info` file. - RecordedForTarball(Info), -} - -#[derive(Clone)] -pub struct Info { - pub commit_date: String, - pub sha: String, - pub short_sha: String, -} - -impl GitInfo { - pub fn new(omit_git_hash: bool, dir: &Path) -> GitInfo { - // See if this even begins to look like a git dir - if !dir.join(".git").exists() { - match read_commit_info_file(dir) { - Some(info) => return GitInfo::RecordedForTarball(info), - None => return GitInfo::Absent, - } - } - - // Make sure git commands work - match Command::new("git").arg("rev-parse").current_dir(dir).output() { - Ok(ref out) if out.status.success() => {} - _ => return GitInfo::Absent, - } - - // If we're ignoring the git info, we don't actually need to collect it, just make sure this - // was a git repo in the first place. - if omit_git_hash { - return GitInfo::Present(None); - } - - // Ok, let's scrape some info - let ver_date = output( - Command::new("git") - .current_dir(dir) - .arg("log") - .arg("-1") - .arg("--date=short") - .arg("--pretty=format:%cd"), - ); - let ver_hash = output(Command::new("git").current_dir(dir).arg("rev-parse").arg("HEAD")); - let short_ver_hash = output( - Command::new("git").current_dir(dir).arg("rev-parse").arg("--short=9").arg("HEAD"), - ); - GitInfo::Present(Some(Info { - commit_date: ver_date.trim().to_string(), - sha: ver_hash.trim().to_string(), - short_sha: short_ver_hash.trim().to_string(), - })) - } - - pub fn info(&self) -> Option<&Info> { - match self { - GitInfo::Absent => None, - GitInfo::Present(info) => info.as_ref(), - GitInfo::RecordedForTarball(info) => Some(info), - } - } - - pub fn sha(&self) -> Option<&str> { - self.info().map(|s| &s.sha[..]) - } - - pub fn sha_short(&self) -> Option<&str> { - self.info().map(|s| &s.short_sha[..]) - } - - pub fn commit_date(&self) -> Option<&str> { - self.info().map(|s| &s.commit_date[..]) - } - - pub fn version(&self, build: &Build, num: &str) -> String { - let mut version = build.release(num); - if let Some(ref inner) = self.info() { - version.push_str(" ("); - version.push_str(&inner.short_sha); - version.push(' '); - version.push_str(&inner.commit_date); - version.push(')'); - } - version - } - - /// Returns whether this directory has a `.git` directory which should be managed by bootstrap. - pub fn is_managed_git_subrepository(&self) -> bool { - match self { - GitInfo::Absent | GitInfo::RecordedForTarball(_) => false, - GitInfo::Present(_) => true, - } - } - - /// Returns whether this is being built from a tarball. - pub fn is_from_tarball(&self) -> bool { - match self { - GitInfo::Absent | GitInfo::Present(_) => false, - GitInfo::RecordedForTarball(_) => true, - } - } -} - -/// Read the commit information from the `git-commit-info` file given the -/// project root. -pub fn read_commit_info_file(root: &Path) -> Option { - if let Ok(contents) = fs::read_to_string(root.join("git-commit-info")) { - let mut lines = contents.lines(); - let sha = lines.next(); - let short_sha = lines.next(); - let commit_date = lines.next(); - let info = match (commit_date, sha, short_sha) { - (Some(commit_date), Some(sha), Some(short_sha)) => Info { - commit_date: commit_date.to_owned(), - sha: sha.to_owned(), - short_sha: short_sha.to_owned(), - }, - _ => panic!("the `git-commit-info` file is malformed"), - }; - Some(info) - } else { - None - } -} - -/// Write the commit information to the `git-commit-info` file given the project -/// root. -pub fn write_commit_info_file(root: &Path, info: &Info) { - let commit_info = format!("{}\n{}\n{}\n", info.sha, info.short_sha, info.commit_date); - t!(fs::write(root.join("git-commit-info"), &commit_info)); -} - -/// Write the commit hash to the `git-commit-hash` file given the project root. -pub fn write_commit_hash_file(root: &Path, sha: &str) { - t!(fs::write(root.join("git-commit-hash"), sha)); -} diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs deleted file mode 100644 index b417abc00..000000000 --- a/src/bootstrap/check.rs +++ /dev/null @@ -1,542 +0,0 @@ -//! Implementation of compiling the compiler and standard library, in "check"-based modes. - -use crate::builder::{crate_description, Alias, Builder, Kind, RunConfig, ShouldRun, Step}; -use crate::cache::Interned; -use crate::compile::{add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo}; -use crate::config::TargetSelection; -use crate::tool::{prepare_tool_cargo, SourceType}; -use crate::INTERNER; -use crate::{Compiler, Mode, Subcommand}; -use std::path::{Path, PathBuf}; - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Std { - pub target: TargetSelection, - /// Whether to build only a subset of crates. - /// - /// This shouldn't be used from other steps; see the comment on [`compile::Rustc`]. - /// - /// [`compile::Rustc`]: crate::compile::Rustc - crates: Interned>, -} - -/// Returns args for the subcommand itself (not for cargo) -fn args(builder: &Builder<'_>) -> Vec { - fn strings<'a>(arr: &'a [&str]) -> impl Iterator + 'a { - arr.iter().copied().map(String::from) - } - - if let Subcommand::Clippy { fix, allow, deny, warn, forbid, .. } = &builder.config.cmd { - // disable the most spammy clippy lints - let ignored_lints = vec![ - "many_single_char_names", // there are a lot in stdarch - "collapsible_if", - "type_complexity", - "missing_safety_doc", // almost 3K warnings - "too_many_arguments", - "needless_lifetimes", // people want to keep the lifetimes - "wrong_self_convention", - ]; - let mut args = vec![]; - if *fix { - #[rustfmt::skip] - args.extend(strings(&[ - "--fix", "-Zunstable-options", - // FIXME: currently, `--fix` gives an error while checking tests for libtest, - // possibly because libtest is not yet built in the sysroot. - // As a workaround, avoid checking tests and benches when passed --fix. - "--lib", "--bins", "--examples", - ])); - } - args.extend(strings(&["--", "--cap-lints", "warn"])); - args.extend(ignored_lints.iter().map(|lint| format!("-Aclippy::{}", lint))); - let mut clippy_lint_levels: Vec = Vec::new(); - allow.iter().for_each(|v| clippy_lint_levels.push(format!("-A{}", v))); - deny.iter().for_each(|v| clippy_lint_levels.push(format!("-D{}", v))); - warn.iter().for_each(|v| clippy_lint_levels.push(format!("-W{}", v))); - forbid.iter().for_each(|v| clippy_lint_levels.push(format!("-F{}", v))); - args.extend(clippy_lint_levels); - args.extend(builder.config.free_args.clone()); - args - } else { - builder.config.free_args.clone() - } -} - -fn cargo_subcommand(kind: Kind) -> &'static str { - match kind { - Kind::Check => "check", - Kind::Clippy => "clippy", - Kind::Fix => "fix", - _ => unreachable!(), - } -} - -impl Std { - pub fn new(target: TargetSelection) -> Self { - Self { target, crates: INTERNER.intern_list(vec![]) } - } -} - -impl Step for Std { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.crate_or_deps("sysroot").path("library") - } - - fn make_run(run: RunConfig<'_>) { - let crates = run.make_run_crates(Alias::Library); - run.builder.ensure(Std { target: run.target, crates }); - } - - fn run(self, builder: &Builder<'_>) { - builder.update_submodule(&Path::new("library").join("stdarch")); - - let target = self.target; - let compiler = builder.compiler(builder.top_stage, builder.config.build); - - let mut cargo = builder.cargo( - compiler, - Mode::Std, - SourceType::InTree, - target, - cargo_subcommand(builder.kind), - ); - std_cargo(builder, target, compiler.stage, &mut cargo); - if matches!(builder.config.cmd, Subcommand::Fix { .. }) { - // By default, cargo tries to fix all targets. Tell it not to fix tests until we've added `test` to the sysroot. - cargo.arg("--lib"); - } - - for krate in &*self.crates { - cargo.arg("-p").arg(krate); - } - - let _guard = builder.msg_check( - format_args!("library artifacts{}", crate_description(&self.crates)), - target, - ); - run_cargo( - builder, - cargo, - args(builder), - &libstd_stamp(builder, compiler, target), - vec![], - true, - false, - ); - - // We skip populating the sysroot in non-zero stage because that'll lead - // to rlib/rmeta conflicts if std gets built during this session. - if compiler.stage == 0 { - let libdir = builder.sysroot_libdir(compiler, target); - let hostdir = builder.sysroot_libdir(compiler, compiler.host); - add_to_sysroot(&builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target)); - } - drop(_guard); - - // don't run on std twice with x.py clippy - // don't check test dependencies if we haven't built libtest - if builder.kind == Kind::Clippy || !self.crates.iter().any(|krate| krate == "test") { - return; - } - - // Then run cargo again, once we've put the rmeta files for the library - // crates into the sysroot. This is needed because e.g., core's tests - // depend on `libtest` -- Cargo presumes it will exist, but it doesn't - // since we initialize with an empty sysroot. - // - // Currently only the "libtest" tree of crates does this. - let mut cargo = builder.cargo( - compiler, - Mode::Std, - SourceType::InTree, - target, - cargo_subcommand(builder.kind), - ); - - // If we're not in stage 0, tests and examples will fail to compile - // from `core` definitions being loaded from two different `libcore` - // .rmeta and .rlib files. - if compiler.stage == 0 { - cargo.arg("--all-targets"); - } - - std_cargo(builder, target, compiler.stage, &mut cargo); - - // Explicitly pass -p for all dependencies krates -- this will force cargo - // to also check the tests/benches/examples for these crates, rather - // than just the leaf crate. - for krate in &*self.crates { - cargo.arg("-p").arg(krate); - } - - let _guard = builder.msg_check("library test/bench/example targets", target); - run_cargo( - builder, - cargo, - args(builder), - &libstd_test_stamp(builder, compiler, target), - vec![], - true, - false, - ); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Rustc { - pub target: TargetSelection, - /// Whether to build only a subset of crates. - /// - /// This shouldn't be used from other steps; see the comment on [`compile::Rustc`]. - /// - /// [`compile::Rustc`]: crate::compile::Rustc - crates: Interned>, -} - -impl Rustc { - pub fn new(target: TargetSelection, builder: &Builder<'_>) -> Self { - let crates = builder - .in_tree_crates("rustc-main", Some(target)) - .into_iter() - .map(|krate| krate.name.to_string()) - .collect(); - Self { target, crates: INTERNER.intern_list(crates) } - } -} - -impl Step for Rustc { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.crate_or_deps("rustc-main").path("compiler") - } - - fn make_run(run: RunConfig<'_>) { - let crates = run.make_run_crates(Alias::Compiler); - run.builder.ensure(Rustc { target: run.target, crates }); - } - - /// Builds the compiler. - /// - /// This will build the compiler for a particular stage of the build using - /// the `compiler` targeting the `target` architecture. The artifacts - /// created will also be linked into the sysroot directory. - fn run(self, builder: &Builder<'_>) { - let compiler = builder.compiler(builder.top_stage, builder.config.build); - let target = self.target; - - if compiler.stage != 0 { - // If we're not in stage 0, then we won't have a std from the beta - // compiler around. That means we need to make sure there's one in - // the sysroot for the compiler to find. Otherwise, we're going to - // fail when building crates that need to generate code (e.g., build - // scripts and their dependencies). - builder.ensure(crate::compile::Std::new(compiler, compiler.host)); - builder.ensure(crate::compile::Std::new(compiler, target)); - } else { - builder.ensure(Std::new(target)); - } - - let mut cargo = builder.cargo( - compiler, - Mode::Rustc, - SourceType::InTree, - target, - cargo_subcommand(builder.kind), - ); - rustc_cargo(builder, &mut cargo, target, compiler.stage); - - // For ./x.py clippy, don't run with --all-targets because - // linting tests and benchmarks can produce very noisy results - if builder.kind != Kind::Clippy { - cargo.arg("--all-targets"); - } - - // Explicitly pass -p for all compiler crates -- this will force cargo - // to also check the tests/benches/examples for these crates, rather - // than just the leaf crate. - for krate in &*self.crates { - cargo.arg("-p").arg(krate); - } - - let _guard = builder.msg_check( - format_args!("compiler artifacts{}", crate_description(&self.crates)), - target, - ); - run_cargo( - builder, - cargo, - args(builder), - &librustc_stamp(builder, compiler, target), - vec![], - true, - false, - ); - - let libdir = builder.sysroot_libdir(compiler, target); - let hostdir = builder.sysroot_libdir(compiler, compiler.host); - add_to_sysroot(&builder, &libdir, &hostdir, &librustc_stamp(builder, compiler, target)); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct CodegenBackend { - pub target: TargetSelection, - pub backend: Interned, -} - -impl Step for CodegenBackend { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.paths(&["compiler/rustc_codegen_cranelift", "compiler/rustc_codegen_gcc"]) - } - - fn make_run(run: RunConfig<'_>) { - for &backend in &[INTERNER.intern_str("cranelift"), INTERNER.intern_str("gcc")] { - run.builder.ensure(CodegenBackend { target: run.target, backend }); - } - } - - fn run(self, builder: &Builder<'_>) { - // FIXME: remove once https://github.com/rust-lang/rust/issues/112393 is resolved - if builder.build.config.vendor && &self.backend == "gcc" { - println!("Skipping checking of `rustc_codegen_gcc` with vendoring enabled."); - return; - } - - let compiler = builder.compiler(builder.top_stage, builder.config.build); - let target = self.target; - let backend = self.backend; - - builder.ensure(Rustc::new(target, builder)); - - let mut cargo = builder.cargo( - compiler, - Mode::Codegen, - SourceType::InTree, - target, - cargo_subcommand(builder.kind), - ); - cargo - .arg("--manifest-path") - .arg(builder.src.join(format!("compiler/rustc_codegen_{backend}/Cargo.toml"))); - rustc_cargo_env(builder, &mut cargo, target, compiler.stage); - - let _guard = builder.msg_check(&backend, target); - - run_cargo( - builder, - cargo, - args(builder), - &codegen_backend_stamp(builder, compiler, target, backend), - vec![], - true, - false, - ); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct RustAnalyzer { - pub target: TargetSelection, -} - -impl Step for RustAnalyzer { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/tools/rust-analyzer").default_condition( - builder - .config - .tools - .as_ref() - .map_or(true, |tools| tools.iter().any(|tool| tool == "rust-analyzer")), - ) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustAnalyzer { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let compiler = builder.compiler(builder.top_stage, builder.config.build); - let target = self.target; - - builder.ensure(Std::new(target)); - - let mut cargo = prepare_tool_cargo( - builder, - compiler, - Mode::ToolStd, - target, - cargo_subcommand(builder.kind), - "src/tools/rust-analyzer", - SourceType::InTree, - &["rust-analyzer/in-rust-tree".to_owned()], - ); - - cargo.allow_features(crate::tool::RustAnalyzer::ALLOW_FEATURES); - - // For ./x.py clippy, don't check those targets because - // linting tests and benchmarks can produce very noisy results - if builder.kind != Kind::Clippy { - // can't use `--all-targets` because `--examples` doesn't work well - cargo.arg("--bins"); - cargo.arg("--tests"); - cargo.arg("--benches"); - } - - let _guard = builder.msg_check("rust-analyzer artifacts", target); - run_cargo( - builder, - cargo, - args(builder), - &stamp(builder, compiler, target), - vec![], - true, - false, - ); - - /// Cargo's output path in a given stage, compiled by a particular - /// compiler for the specified target. - fn stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { - builder.cargo_out(compiler, Mode::ToolStd, target).join(".rust-analyzer-check.stamp") - } - } -} - -macro_rules! tool_check_step { - ($name:ident, $path:literal, $($alias:literal, )* $source_type:path $(, $default:literal )?) => { - #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] - pub struct $name { - pub target: TargetSelection, - } - - impl Step for $name { - type Output = (); - const ONLY_HOSTS: bool = true; - // don't ever check out-of-tree tools by default, they'll fail when toolstate is broken - const DEFAULT: bool = matches!($source_type, SourceType::InTree) $( && $default )?; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.paths(&[ $path, $($alias),* ]) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure($name { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let compiler = builder.compiler(builder.top_stage, builder.config.build); - let target = self.target; - - builder.ensure(Rustc::new(target, builder)); - - let mut cargo = prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - target, - cargo_subcommand(builder.kind), - $path, - $source_type, - &[], - ); - - // For ./x.py clippy, don't run with --all-targets because - // linting tests and benchmarks can produce very noisy results - if builder.kind != Kind::Clippy { - cargo.arg("--all-targets"); - } - - // Enable internal lints for clippy and rustdoc - // NOTE: this doesn't enable lints for any other tools unless they explicitly add `#![warn(rustc::internal)]` - // See https://github.com/rust-lang/rust/pull/80573#issuecomment-754010776 - cargo.rustflag("-Zunstable-options"); - let _guard = builder.msg_check(&concat!(stringify!($name), " artifacts").to_lowercase(), target); - run_cargo( - builder, - cargo, - args(builder), - &stamp(builder, compiler, target), - vec![], - true, - false, - ); - - /// Cargo's output path in a given stage, compiled by a particular - /// compiler for the specified target. - fn stamp( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, - ) -> PathBuf { - builder - .cargo_out(compiler, Mode::ToolRustc, target) - .join(format!(".{}-check.stamp", stringify!($name).to_lowercase())) - } - } - } - }; -} - -tool_check_step!(Rustdoc, "src/tools/rustdoc", "src/librustdoc", SourceType::InTree); -// Clippy, miri and Rustfmt are hybrids. They are external tools, but use a git subtree instead -// of a submodule. Since the SourceType only drives the deny-warnings -// behavior, treat it as in-tree so that any new warnings in clippy will be -// rejected. -tool_check_step!(Clippy, "src/tools/clippy", SourceType::InTree); -tool_check_step!(Miri, "src/tools/miri", SourceType::InTree); -tool_check_step!(CargoMiri, "src/tools/miri/cargo-miri", SourceType::InTree); -tool_check_step!(Rls, "src/tools/rls", SourceType::InTree); -tool_check_step!(Rustfmt, "src/tools/rustfmt", SourceType::InTree); -tool_check_step!(MiroptTestTools, "src/tools/miropt-test-tools", SourceType::InTree); - -tool_check_step!(Bootstrap, "src/bootstrap", SourceType::InTree, false); - -/// Cargo's output path for the standard library in a given stage, compiled -/// by a particular compiler for the specified target. -fn libstd_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { - builder.cargo_out(compiler, Mode::Std, target).join(".libstd-check.stamp") -} - -/// Cargo's output path for the standard library in a given stage, compiled -/// by a particular compiler for the specified target. -fn libstd_test_stamp( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, -) -> PathBuf { - builder.cargo_out(compiler, Mode::Std, target).join(".libstd-check-test.stamp") -} - -/// Cargo's output path for librustc in a given stage, compiled by a particular -/// compiler for the specified target. -fn librustc_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { - builder.cargo_out(compiler, Mode::Rustc, target).join(".librustc-check.stamp") -} - -/// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular -/// compiler for the specified target and backend. -fn codegen_backend_stamp( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, - backend: Interned, -) -> PathBuf { - builder - .cargo_out(compiler, Mode::Codegen, target) - .join(format!(".librustc_codegen_{backend}-check.stamp")) -} diff --git a/src/bootstrap/clean.rs b/src/bootstrap/clean.rs deleted file mode 100644 index 7389816b4..000000000 --- a/src/bootstrap/clean.rs +++ /dev/null @@ -1,237 +0,0 @@ -//! Implementation of `make clean` in rustbuild. -//! -//! Responsible for cleaning out a build directory of all old and stale -//! artifacts to prepare for a fresh build. Currently doesn't remove the -//! `build/cache` directory (download cache) or the `build/$target/llvm` -//! directory unless the `--all` flag is present. - -use std::fs; -use std::io::{self, ErrorKind}; -use std::path::Path; - -use crate::builder::{crate_description, Builder, RunConfig, ShouldRun, Step}; -use crate::cache::Interned; -use crate::util::t; -use crate::{Build, Compiler, Mode, Subcommand}; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct CleanAll {} - -impl Step for CleanAll { - const DEFAULT: bool = true; - type Output = (); - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(CleanAll {}) - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - let Subcommand::Clean { all, stage } = builder.config.cmd else { - unreachable!("wrong subcommand?") - }; - - if all && stage.is_some() { - panic!("--all and --stage can't be used at the same time for `x clean`"); - } - - clean(builder.build, all, stage) - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() // handled by DEFAULT - } -} - -macro_rules! clean_crate_tree { - ( $( $name:ident, $mode:path, $root_crate:literal);+ $(;)? ) => { $( - #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] - pub struct $name { - compiler: Compiler, - crates: Interned>, - } - - impl Step for $name { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let crates = run.builder.in_tree_crates($root_crate, None); - run.crates(crates) - } - - fn make_run(run: RunConfig<'_>) { - let builder = run.builder; - let compiler = builder.compiler(builder.top_stage, run.target); - builder.ensure(Self { crates: run.cargo_crates_in_set(), compiler }); - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - let compiler = self.compiler; - let target = compiler.host; - let mut cargo = builder.bare_cargo(compiler, $mode, target, "clean"); - for krate in &*self.crates { - cargo.arg("-p"); - cargo.arg(krate); - } - - builder.info(&format!( - "Cleaning{} stage{} {} artifacts ({} -> {})", - crate_description(&self.crates), compiler.stage, stringify!($name).to_lowercase(), &compiler.host, target, - )); - - // NOTE: doesn't use `run_cargo` because we don't want to save a stamp file, - // and doesn't use `stream_cargo` to avoid passing `--message-format` which `clean` doesn't accept. - builder.run(&mut cargo); - } - } - )+ } -} - -clean_crate_tree! { - Rustc, Mode::Rustc, "rustc-main"; - Std, Mode::Std, "sysroot"; -} - -fn clean(build: &Build, all: bool, stage: Option) { - if build.config.dry_run() { - return; - } - - rm_rf("tmp".as_ref()); - - // Clean the entire build directory - if all { - rm_rf(&build.out); - return; - } - - // Clean the target stage artifacts - if let Some(stage) = stage { - clean_specific_stage(build, stage); - return; - } - - // Follow the default behaviour - clean_default(build); -} - -fn clean_specific_stage(build: &Build, stage: u32) { - for host in &build.hosts { - let entries = match build.out.join(host.triple).read_dir() { - Ok(iter) => iter, - Err(_) => continue, - }; - - for entry in entries { - let entry = t!(entry); - let stage_prefix = format!("stage{}", stage); - - // if current entry is not related with the target stage, continue - if !entry.file_name().to_str().unwrap_or("").contains(&stage_prefix) { - continue; - } - - let path = t!(entry.path().canonicalize()); - rm_rf(&path); - } - } -} - -fn clean_default(build: &Build) { - rm_rf(&build.out.join("tmp")); - rm_rf(&build.out.join("dist")); - rm_rf(&build.out.join("bootstrap")); - rm_rf(&build.out.join("rustfmt.stamp")); - - for host in &build.hosts { - let entries = match build.out.join(host.triple).read_dir() { - Ok(iter) => iter, - Err(_) => continue, - }; - - for entry in entries { - let entry = t!(entry); - if entry.file_name().to_str() == Some("llvm") { - continue; - } - let path = t!(entry.path().canonicalize()); - rm_rf(&path); - } - } -} - -fn rm_rf(path: &Path) { - match path.symlink_metadata() { - Err(e) => { - if e.kind() == ErrorKind::NotFound { - return; - } - panic!("failed to get metadata for file {}: {}", path.display(), e); - } - Ok(metadata) => { - if metadata.file_type().is_file() || metadata.file_type().is_symlink() { - do_op(path, "remove file", |p| { - fs::remove_file(p).or_else(|e| { - // Work around the fact that we cannot - // delete an executable while it runs on Windows. - #[cfg(windows)] - if e.kind() == std::io::ErrorKind::PermissionDenied - && p.file_name().and_then(std::ffi::OsStr::to_str) - == Some("bootstrap.exe") - { - eprintln!("warning: failed to delete '{}'.", p.display()); - return Ok(()); - } - Err(e) - }) - }); - return; - } - - for file in t!(fs::read_dir(path)) { - rm_rf(&t!(file).path()); - } - do_op(path, "remove dir", |p| { - fs::remove_dir(p).or_else(|e| { - // Check for dir not empty on Windows - // FIXME: Once `ErrorKind::DirectoryNotEmpty` is stabilized, - // match on `e.kind()` instead. - #[cfg(windows)] - if e.raw_os_error() == Some(145) { - return Ok(()); - } - - Err(e) - }) - }); - } - }; -} - -fn do_op(path: &Path, desc: &str, mut f: F) -where - F: FnMut(&Path) -> io::Result<()>, -{ - match f(path) { - Ok(()) => {} - // On windows we can't remove a readonly file, and git will often clone files as readonly. - // As a result, we have some special logic to remove readonly files on windows. - // This is also the reason that we can't use things like fs::remove_dir_all(). - Err(ref e) if cfg!(windows) && e.kind() == ErrorKind::PermissionDenied => { - let m = t!(path.symlink_metadata()); - let mut p = m.permissions(); - p.set_readonly(false); - t!(fs::set_permissions(path, p)); - f(path).unwrap_or_else(|e| { - // Delete symlinked directories on Windows - #[cfg(windows)] - if m.file_type().is_symlink() && path.is_dir() && fs::remove_dir(path).is_ok() { - return; - } - panic!("failed to {} {}: {}", desc, path.display(), e); - }); - } - Err(e) => { - panic!("failed to {} {}: {}", desc, path.display(), e); - } - } -} diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs deleted file mode 100644 index 292ccc578..000000000 --- a/src/bootstrap/compile.rs +++ /dev/null @@ -1,2015 +0,0 @@ -//! Implementation of compiling various phases of the compiler and standard -//! library. -//! -//! This module contains some of the real meat in the rustbuild build system -//! which is where Cargo is used to compile the standard library, libtest, and -//! the compiler. This module is also responsible for assembling the sysroot as it -//! goes along from the output of the previous stage. - -use std::borrow::Cow; -use std::collections::HashSet; -use std::env; -use std::ffi::OsStr; -use std::fs; -use std::io::prelude::*; -use std::io::BufReader; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use std::str; - -use serde_derive::Deserialize; - -use crate::builder::crate_description; -use crate::builder::Cargo; -use crate::builder::{Builder, Kind, PathSet, RunConfig, ShouldRun, Step, TaskPath}; -use crate::cache::{Interned, INTERNER}; -use crate::config::{DebuginfoLevel, LlvmLibunwind, RustcLto, TargetSelection}; -use crate::dist; -use crate::llvm; -use crate::tool::SourceType; -use crate::util::get_clang_cl_resource_dir; -use crate::util::{exe, is_debug_info, is_dylib, output, symlink_dir, t, up_to_date}; -use crate::LLVM_TOOLS; -use crate::{CLang, Compiler, DependencyType, GitRepo, Mode}; -use filetime::FileTime; - -#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct Std { - pub target: TargetSelection, - pub compiler: Compiler, - /// Whether to build only a subset of crates in the standard library. - /// - /// This shouldn't be used from other steps; see the comment on [`Rustc`]. - crates: Interned>, - /// When using download-rustc, we need to use a new build of `std` for running unit tests of Std itself, - /// but we need to use the downloaded copy of std for linking to rustdoc. Allow this to be overriden by `builder.ensure` from other steps. - force_recompile: bool, -} - -impl Std { - pub fn new(compiler: Compiler, target: TargetSelection) -> Self { - Self { target, compiler, crates: Default::default(), force_recompile: false } - } - - pub fn force_recompile(compiler: Compiler, target: TargetSelection) -> Self { - Self { target, compiler, crates: Default::default(), force_recompile: true } - } -} - -impl Step for Std { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - // When downloading stage1, the standard library has already been copied to the sysroot, so - // there's no need to rebuild it. - let builder = run.builder; - run.crate_or_deps("sysroot") - .path("library") - .lazy_default_condition(Box::new(|| !builder.download_rustc())) - } - - fn make_run(run: RunConfig<'_>) { - // If the paths include "library", build the entire standard library. - let has_alias = - run.paths.iter().any(|set| set.assert_single_path().path.ends_with("library")); - let crates = if has_alias { Default::default() } else { run.cargo_crates_in_set() }; - - run.builder.ensure(Std { - compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), - target: run.target, - crates, - force_recompile: false, - }); - } - - /// Builds the standard library. - /// - /// This will build the standard library for a particular stage of the build - /// using the `compiler` targeting the `target` architecture. The artifacts - /// created will also be linked into the sysroot directory. - fn run(self, builder: &Builder<'_>) { - let target = self.target; - let compiler = self.compiler; - - // When using `download-rustc`, we already have artifacts for the host available. Don't - // recompile them. - if builder.download_rustc() && target == builder.build.build - // NOTE: the beta compiler may generate different artifacts than the downloaded compiler, so - // its artifacts can't be reused. - && compiler.stage != 0 - // This check is specific to testing std itself; see `test::Std` for more details. - && !self.force_recompile - { - cp_rustc_component_to_ci_sysroot( - builder, - compiler, - builder.config.ci_rust_std_contents(), - ); - return; - } - - if builder.config.keep_stage.contains(&compiler.stage) - || builder.config.keep_stage_std.contains(&compiler.stage) - { - builder.info("Warning: Using a potentially old libstd. This may not behave well."); - - copy_third_party_objects(builder, &compiler, target); - copy_self_contained_objects(builder, &compiler, target); - - builder.ensure(StdLink::from_std(self, compiler)); - return; - } - - builder.update_submodule(&Path::new("library").join("stdarch")); - - // Profiler information requires LLVM's compiler-rt - if builder.config.profiler { - builder.update_submodule(&Path::new("src/llvm-project")); - } - - let mut target_deps = builder.ensure(StartupObjects { compiler, target }); - - let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); - if compiler_to_use != compiler { - builder.ensure(Std::new(compiler_to_use, target)); - let msg = if compiler_to_use.host == target { - format!( - "Uplifting library (stage{} -> stage{})", - compiler_to_use.stage, compiler.stage - ) - } else { - format!( - "Uplifting library (stage{}:{} -> stage{}:{})", - compiler_to_use.stage, compiler_to_use.host, compiler.stage, target - ) - }; - builder.info(&msg); - - // Even if we're not building std this stage, the new sysroot must - // still contain the third party objects needed by various targets. - copy_third_party_objects(builder, &compiler, target); - copy_self_contained_objects(builder, &compiler, target); - - builder.ensure(StdLink::from_std(self, compiler_to_use)); - return; - } - - target_deps.extend(copy_third_party_objects(builder, &compiler, target)); - target_deps.extend(copy_self_contained_objects(builder, &compiler, target)); - - let mut cargo = builder.cargo(compiler, Mode::Std, SourceType::InTree, target, "build"); - std_cargo(builder, target, compiler.stage, &mut cargo); - for krate in &*self.crates { - cargo.arg("-p").arg(krate); - } - - // See src/bootstrap/synthetic_targets.rs - if target.is_synthetic() { - cargo.env("RUSTC_BOOTSTRAP_SYNTHETIC_TARGET", "1"); - } - - let _guard = builder.msg( - Kind::Build, - compiler.stage, - format_args!("library artifacts{}", crate_description(&self.crates)), - compiler.host, - target, - ); - run_cargo( - builder, - cargo, - vec![], - &libstd_stamp(builder, compiler, target), - target_deps, - false, - false, - ); - - builder.ensure(StdLink::from_std( - self, - builder.compiler(compiler.stage, builder.config.build), - )); - } -} - -fn copy_and_stamp( - builder: &Builder<'_>, - libdir: &Path, - sourcedir: &Path, - name: &str, - target_deps: &mut Vec<(PathBuf, DependencyType)>, - dependency_type: DependencyType, -) { - let target = libdir.join(name); - builder.copy(&sourcedir.join(name), &target); - - target_deps.push((target, dependency_type)); -} - -fn copy_llvm_libunwind(builder: &Builder<'_>, target: TargetSelection, libdir: &Path) -> PathBuf { - let libunwind_path = builder.ensure(llvm::Libunwind { target }); - let libunwind_source = libunwind_path.join("libunwind.a"); - let libunwind_target = libdir.join("libunwind.a"); - builder.copy(&libunwind_source, &libunwind_target); - libunwind_target -} - -/// Copies third party objects needed by various targets. -fn copy_third_party_objects( - builder: &Builder<'_>, - compiler: &Compiler, - target: TargetSelection, -) -> Vec<(PathBuf, DependencyType)> { - let mut target_deps = vec![]; - - if builder.config.sanitizers_enabled(target) && compiler.stage != 0 { - // The sanitizers are only copied in stage1 or above, - // to avoid creating dependency on LLVM. - target_deps.extend( - copy_sanitizers(builder, &compiler, target) - .into_iter() - .map(|d| (d, DependencyType::Target)), - ); - } - - if target == "x86_64-fortanix-unknown-sgx" - || builder.config.llvm_libunwind(target) == LlvmLibunwind::InTree - && (target.contains("linux") || target.contains("fuchsia")) - { - let libunwind_path = - copy_llvm_libunwind(builder, target, &builder.sysroot_libdir(*compiler, target)); - target_deps.push((libunwind_path, DependencyType::Target)); - } - - target_deps -} - -/// Copies third party objects needed by various targets for self-contained linkage. -fn copy_self_contained_objects( - builder: &Builder<'_>, - compiler: &Compiler, - target: TargetSelection, -) -> Vec<(PathBuf, DependencyType)> { - let libdir_self_contained = builder.sysroot_libdir(*compiler, target).join("self-contained"); - t!(fs::create_dir_all(&libdir_self_contained)); - let mut target_deps = vec![]; - - // Copies the libc and CRT objects. - // - // rustc historically provides a more self-contained installation for musl targets - // not requiring the presence of a native musl toolchain. For example, it can fall back - // to using gcc from a glibc-targeting toolchain for linking. - // To do that we have to distribute musl startup objects as a part of Rust toolchain - // and link with them manually in the self-contained mode. - if target.contains("musl") && !target.contains("unikraft") { - let srcdir = builder.musl_libdir(target).unwrap_or_else(|| { - panic!("Target {:?} does not have a \"musl-libdir\" key", target.triple) - }); - for &obj in &["libc.a", "crt1.o", "Scrt1.o", "rcrt1.o", "crti.o", "crtn.o"] { - copy_and_stamp( - builder, - &libdir_self_contained, - &srcdir, - obj, - &mut target_deps, - DependencyType::TargetSelfContained, - ); - } - let crt_path = builder.ensure(llvm::CrtBeginEnd { target }); - for &obj in &["crtbegin.o", "crtbeginS.o", "crtend.o", "crtendS.o"] { - let src = crt_path.join(obj); - let target = libdir_self_contained.join(obj); - builder.copy(&src, &target); - target_deps.push((target, DependencyType::TargetSelfContained)); - } - - if !target.starts_with("s390x") { - let libunwind_path = copy_llvm_libunwind(builder, target, &libdir_self_contained); - target_deps.push((libunwind_path, DependencyType::TargetSelfContained)); - } - } else if target.contains("-wasi") { - let srcdir = builder - .wasi_root(target) - .unwrap_or_else(|| { - panic!("Target {:?} does not have a \"wasi-root\" key", target.triple) - }) - .join("lib") - .join(target.to_string().replace("-preview1", "")); - for &obj in &["libc.a", "crt1-command.o", "crt1-reactor.o"] { - copy_and_stamp( - builder, - &libdir_self_contained, - &srcdir, - obj, - &mut target_deps, - DependencyType::TargetSelfContained, - ); - } - } else if target.ends_with("windows-gnu") { - for obj in ["crt2.o", "dllcrt2.o"].iter() { - let src = compiler_file(builder, &builder.cc(target), target, CLang::C, obj); - let target = libdir_self_contained.join(obj); - builder.copy(&src, &target); - target_deps.push((target, DependencyType::TargetSelfContained)); - } - } - - target_deps -} - -/// Configure cargo to compile the standard library, adding appropriate env vars -/// and such. -pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, cargo: &mut Cargo) { - if let Some(target) = env::var_os("MACOSX_STD_DEPLOYMENT_TARGET") { - cargo.env("MACOSX_DEPLOYMENT_TARGET", target); - } - - if let Some(path) = builder.config.profiler_path(target) { - cargo.env("LLVM_PROFILER_RT_LIB", path); - } - - // Determine if we're going to compile in optimized C intrinsics to - // the `compiler-builtins` crate. These intrinsics live in LLVM's - // `compiler-rt` repository, but our `src/llvm-project` submodule isn't - // always checked out, so we need to conditionally look for this. (e.g. if - // an external LLVM is used we skip the LLVM submodule checkout). - // - // Note that this shouldn't affect the correctness of `compiler-builtins`, - // but only its speed. Some intrinsics in C haven't been translated to Rust - // yet but that's pretty rare. Other intrinsics have optimized - // implementations in C which have only had slower versions ported to Rust, - // so we favor the C version where we can, but it's not critical. - // - // If `compiler-rt` is available ensure that the `c` feature of the - // `compiler-builtins` crate is enabled and it's configured to learn where - // `compiler-rt` is located. - let compiler_builtins_root = builder.src.join("src/llvm-project/compiler-rt"); - let compiler_builtins_c_feature = if compiler_builtins_root.exists() { - // Note that `libprofiler_builtins/build.rs` also computes this so if - // you're changing something here please also change that. - cargo.env("RUST_COMPILER_RT_ROOT", &compiler_builtins_root); - " compiler-builtins-c" - } else { - "" - }; - - // `libtest` uses this to know whether or not to support - // `-Zunstable-options`. - if !builder.unstable_features() { - cargo.env("CFG_DISABLE_UNSTABLE_FEATURES", "1"); - } - - let mut features = String::new(); - - // Cranelift doesn't support `asm`. - if stage != 0 && builder.config.default_codegen_backend().unwrap_or_default() == "cranelift" { - features += " compiler-builtins-no-asm"; - } - - if builder.no_std(target) == Some(true) { - features += " compiler-builtins-mem"; - if !target.starts_with("bpf") { - features.push_str(compiler_builtins_c_feature); - } - - // for no-std targets we only compile a few no_std crates - cargo - .args(&["-p", "alloc"]) - .arg("--manifest-path") - .arg(builder.src.join("library/alloc/Cargo.toml")) - .arg("--features") - .arg(features); - } else { - features += &builder.std_features(target); - features.push_str(compiler_builtins_c_feature); - - cargo - .arg("--features") - .arg(features) - .arg("--manifest-path") - .arg(builder.src.join("library/sysroot/Cargo.toml")); - - // Help the libc crate compile by assisting it in finding various - // sysroot native libraries. - if target.contains("musl") { - if let Some(p) = builder.musl_libdir(target) { - let root = format!("native={}", p.to_str().unwrap()); - cargo.rustflag("-L").rustflag(&root); - } - } - - if target.contains("-wasi") { - if let Some(p) = builder.wasi_root(target) { - let root = format!( - "native={}/lib/{}", - p.to_str().unwrap(), - target.to_string().replace("-preview1", "") - ); - cargo.rustflag("-L").rustflag(&root); - } - } - } - - // By default, rustc uses `-Cembed-bitcode=yes`, and Cargo overrides that - // with `-Cembed-bitcode=no` for non-LTO builds. However, libstd must be - // built with bitcode so that the produced rlibs can be used for both LTO - // builds (which use bitcode) and non-LTO builds (which use object code). - // So we override the override here! - // - // But we don't bother for the stage 0 compiler because it's never used - // with LTO. - if stage >= 1 { - cargo.rustflag("-Cembed-bitcode=yes"); - } - if builder.config.rust_lto == RustcLto::Off { - cargo.rustflag("-Clto=off"); - } - - // By default, rustc does not include unwind tables unless they are required - // for a particular target. They are not required by RISC-V targets, but - // compiling the standard library with them means that users can get - // backtraces without having to recompile the standard library themselves. - // - // This choice was discussed in https://github.com/rust-lang/rust/pull/69890 - if target.contains("riscv") { - cargo.rustflag("-Cforce-unwind-tables=yes"); - } - - let html_root = - format!("-Zcrate-attr=doc(html_root_url=\"{}/\")", builder.doc_rust_lang_org_channel(),); - cargo.rustflag(&html_root); - cargo.rustdocflag(&html_root); - - cargo.rustdocflag("-Zcrate-attr=warn(rust_2018_idioms)"); -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -struct StdLink { - pub compiler: Compiler, - pub target_compiler: Compiler, - pub target: TargetSelection, - /// Not actually used; only present to make sure the cache invalidation is correct. - crates: Interned>, - /// See [`Std::force_recompile`]. - force_recompile: bool, -} - -impl StdLink { - fn from_std(std: Std, host_compiler: Compiler) -> Self { - Self { - compiler: host_compiler, - target_compiler: std.compiler, - target: std.target, - crates: std.crates, - force_recompile: std.force_recompile, - } - } -} - -impl Step for StdLink { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Link all libstd rlibs/dylibs into the sysroot location. - /// - /// Links those artifacts generated by `compiler` to the `stage` compiler's - /// sysroot for the specified `host` and `target`. - /// - /// Note that this assumes that `compiler` has already generated the libstd - /// libraries for `target`, and this method will find them in the relevant - /// output directory. - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let target_compiler = self.target_compiler; - let target = self.target; - - // NOTE: intentionally does *not* check `target == builder.build` to avoid having to add the same check in `test::Crate`. - let (libdir, hostdir) = if self.force_recompile && builder.download_rustc() { - // NOTE: copies part of `sysroot_libdir` to avoid having to add a new `force_recompile` argument there too - let lib = builder.sysroot_libdir_relative(self.compiler); - let sysroot = builder.ensure(crate::compile::Sysroot { - compiler: self.compiler, - force_recompile: self.force_recompile, - }); - let libdir = sysroot.join(lib).join("rustlib").join(target.triple).join("lib"); - let hostdir = sysroot.join(lib).join("rustlib").join(compiler.host.triple).join("lib"); - (INTERNER.intern_path(libdir), INTERNER.intern_path(hostdir)) - } else { - let libdir = builder.sysroot_libdir(target_compiler, target); - let hostdir = builder.sysroot_libdir(target_compiler, compiler.host); - (libdir, hostdir) - }; - - add_to_sysroot(builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target)); - - // Special case for stage0, to make `rustup toolchain link` and `x dist --stage 0` - // work for stage0-sysroot. We only do this if the stage0 compiler comes from beta, - // and is not set to a custom path. - if compiler.stage == 0 - && builder - .build - .config - .initial_rustc - .starts_with(builder.out.join(&compiler.host.triple).join("stage0/bin")) - { - // Copy bin files from stage0/bin to stage0-sysroot/bin - let sysroot = builder.out.join(&compiler.host.triple).join("stage0-sysroot"); - - let host = compiler.host.triple; - let stage0_bin_dir = builder.out.join(&host).join("stage0/bin"); - let sysroot_bin_dir = sysroot.join("bin"); - t!(fs::create_dir_all(&sysroot_bin_dir)); - builder.cp_r(&stage0_bin_dir, &sysroot_bin_dir); - - // Copy all *.so files from stage0/lib to stage0-sysroot/lib - let stage0_lib_dir = builder.out.join(&host).join("stage0/lib"); - if let Ok(files) = fs::read_dir(&stage0_lib_dir) { - for file in files { - let file = t!(file); - let path = file.path(); - if path.is_file() && is_dylib(&file.file_name().into_string().unwrap()) { - builder.copy(&path, &sysroot.join("lib").join(path.file_name().unwrap())); - } - } - } - - // Copy codegen-backends from stage0 - let sysroot_codegen_backends = builder.sysroot_codegen_backends(compiler); - t!(fs::create_dir_all(&sysroot_codegen_backends)); - let stage0_codegen_backends = builder - .out - .join(&host) - .join("stage0/lib/rustlib") - .join(&host) - .join("codegen-backends"); - builder.cp_r(&stage0_codegen_backends, &sysroot_codegen_backends); - } - } -} - -/// Copies sanitizer runtime libraries into target libdir. -fn copy_sanitizers( - builder: &Builder<'_>, - compiler: &Compiler, - target: TargetSelection, -) -> Vec { - let runtimes: Vec = builder.ensure(llvm::Sanitizers { target }); - - if builder.config.dry_run() { - return Vec::new(); - } - - let mut target_deps = Vec::new(); - let libdir = builder.sysroot_libdir(*compiler, target); - - for runtime in &runtimes { - let dst = libdir.join(&runtime.name); - builder.copy(&runtime.path, &dst); - - // The `aarch64-apple-ios-macabi` and `x86_64-apple-ios-macabi` are also supported for - // sanitizers, but they share a sanitizer runtime with `${arch}-apple-darwin`, so we do - // not list them here to rename and sign the runtime library. - if target == "x86_64-apple-darwin" - || target == "aarch64-apple-darwin" - || target == "aarch64-apple-ios" - || target == "aarch64-apple-ios-sim" - || target == "x86_64-apple-ios" - { - // Update the library’s install name to reflect that it has been renamed. - apple_darwin_update_library_name(&dst, &format!("@rpath/{}", &runtime.name)); - // Upon renaming the install name, the code signature of the file will invalidate, - // so we will sign it again. - apple_darwin_sign_file(&dst); - } - - target_deps.push(dst); - } - - target_deps -} - -fn apple_darwin_update_library_name(library_path: &Path, new_name: &str) { - let status = Command::new("install_name_tool") - .arg("-id") - .arg(new_name) - .arg(library_path) - .status() - .expect("failed to execute `install_name_tool`"); - assert!(status.success()); -} - -fn apple_darwin_sign_file(file_path: &Path) { - let status = Command::new("codesign") - .arg("-f") // Force to rewrite the existing signature - .arg("-s") - .arg("-") - .arg(file_path) - .status() - .expect("failed to execute `codesign`"); - assert!(status.success()); -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct StartupObjects { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for StartupObjects { - type Output = Vec<(PathBuf, DependencyType)>; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("library/rtstartup") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(StartupObjects { - compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), - target: run.target, - }); - } - - /// Builds and prepare startup objects like rsbegin.o and rsend.o - /// - /// These are primarily used on Windows right now for linking executables/dlls. - /// They don't require any library support as they're just plain old object - /// files, so we just use the nightly snapshot compiler to always build them (as - /// no other compilers are guaranteed to be available). - fn run(self, builder: &Builder<'_>) -> Vec<(PathBuf, DependencyType)> { - let for_compiler = self.compiler; - let target = self.target; - if !target.ends_with("windows-gnu") { - return vec![]; - } - - let mut target_deps = vec![]; - - let src_dir = &builder.src.join("library").join("rtstartup"); - let dst_dir = &builder.native_dir(target).join("rtstartup"); - let sysroot_dir = &builder.sysroot_libdir(for_compiler, target); - t!(fs::create_dir_all(dst_dir)); - - for file in &["rsbegin", "rsend"] { - let src_file = &src_dir.join(file.to_string() + ".rs"); - let dst_file = &dst_dir.join(file.to_string() + ".o"); - if !up_to_date(src_file, dst_file) { - let mut cmd = Command::new(&builder.initial_rustc); - cmd.env("RUSTC_BOOTSTRAP", "1"); - if !builder.local_rebuild { - // a local_rebuild compiler already has stage1 features - cmd.arg("--cfg").arg("bootstrap"); - } - builder.run( - cmd.arg("--target") - .arg(target.rustc_target_arg()) - .arg("--emit=obj") - .arg("-o") - .arg(dst_file) - .arg(src_file), - ); - } - - let target = sysroot_dir.join((*file).to_string() + ".o"); - builder.copy(dst_file, &target); - target_deps.push((target, DependencyType::Target)); - } - - target_deps - } -} - -fn cp_rustc_component_to_ci_sysroot( - builder: &Builder<'_>, - compiler: Compiler, - contents: Vec, -) { - let sysroot = builder.ensure(Sysroot { compiler, force_recompile: false }); - let ci_rustc_dir = builder.config.ci_rustc_dir(); - - for file in contents { - let src = ci_rustc_dir.join(&file); - let dst = sysroot.join(file); - if src.is_dir() { - t!(fs::create_dir_all(dst)); - } else { - builder.copy(&src, &dst); - } - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Rustc { - pub target: TargetSelection, - pub compiler: Compiler, - /// Whether to build a subset of crates, rather than the whole compiler. - /// - /// This should only be requested by the user, not used within rustbuild itself. - /// Using it within rustbuild can lead to confusing situation where lints are replayed - /// in two different steps. - crates: Interned>, -} - -impl Rustc { - pub fn new(compiler: Compiler, target: TargetSelection) -> Self { - Self { target, compiler, crates: Default::default() } - } -} - -impl Step for Rustc { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let mut crates = run.builder.in_tree_crates("rustc-main", None); - for (i, krate) in crates.iter().enumerate() { - // We can't allow `build rustc` as an alias for this Step, because that's reserved by `Assemble`. - // Ideally Assemble would use `build compiler` instead, but that seems too confusing to be worth the breaking change. - if krate.name == "rustc-main" { - crates.swap_remove(i); - break; - } - } - run.crates(crates) - } - - fn make_run(run: RunConfig<'_>) { - let crates = run.cargo_crates_in_set(); - run.builder.ensure(Rustc { - compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), - target: run.target, - crates, - }); - } - - /// Builds the compiler. - /// - /// This will build the compiler for a particular stage of the build using - /// the `compiler` targeting the `target` architecture. The artifacts - /// created will also be linked into the sysroot directory. - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let target = self.target; - - // NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler, - // so its artifacts can't be reused. - if builder.download_rustc() && compiler.stage != 0 { - // Copy the existing artifacts instead of rebuilding them. - // NOTE: this path is only taken for tools linking to rustc-dev (including ui-fulldeps tests). - cp_rustc_component_to_ci_sysroot( - builder, - compiler, - builder.config.ci_rustc_dev_contents(), - ); - return; - } - - builder.ensure(Std::new(compiler, target)); - - if builder.config.keep_stage.contains(&compiler.stage) { - builder.info("Warning: Using a potentially old librustc. This may not behave well."); - builder.info("Warning: Use `--keep-stage-std` if you want to rebuild the compiler when it changes"); - builder.ensure(RustcLink::from_rustc(self, compiler)); - return; - } - - let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); - if compiler_to_use != compiler { - builder.ensure(Rustc::new(compiler_to_use, target)); - let msg = if compiler_to_use.host == target { - format!( - "Uplifting rustc (stage{} -> stage{})", - compiler_to_use.stage, - compiler.stage + 1 - ) - } else { - format!( - "Uplifting rustc (stage{}:{} -> stage{}:{})", - compiler_to_use.stage, - compiler_to_use.host, - compiler.stage + 1, - target - ) - }; - builder.info(&msg); - builder.ensure(RustcLink::from_rustc(self, compiler_to_use)); - return; - } - - // Ensure that build scripts and proc macros have a std / libproc_macro to link against. - builder.ensure(Std::new( - builder.compiler(self.compiler.stage, builder.config.build), - builder.config.build, - )); - - let mut cargo = builder.cargo(compiler, Mode::Rustc, SourceType::InTree, target, "build"); - rustc_cargo(builder, &mut cargo, target, compiler.stage); - - if builder.config.rust_profile_use.is_some() - && builder.config.rust_profile_generate.is_some() - { - panic!("Cannot use and generate PGO profiles at the same time"); - } - - // With LLD, we can use ICF (identical code folding) to reduce the executable size - // of librustc_driver/rustc and to improve i-cache utilization. - // - // -Wl,[link options] doesn't work on MSVC. However, /OPT:ICF (technically /OPT:REF,ICF) - // is already on by default in MSVC optimized builds, which is interpreted as --icf=all: - // https://github.com/llvm/llvm-project/blob/3329cec2f79185bafd678f310fafadba2a8c76d2/lld/COFF/Driver.cpp#L1746 - // https://github.com/rust-lang/rust/blob/f22819bcce4abaff7d1246a56eec493418f9f4ee/compiler/rustc_codegen_ssa/src/back/linker.rs#L827 - if builder.config.use_lld && !compiler.host.contains("msvc") { - cargo.rustflag("-Clink-args=-Wl,--icf=all"); - } - - let is_collecting = if let Some(path) = &builder.config.rust_profile_generate { - if compiler.stage == 1 { - cargo.rustflag(&format!("-Cprofile-generate={path}")); - // Apparently necessary to avoid overflowing the counters during - // a Cargo build profile - cargo.rustflag("-Cllvm-args=-vp-counters-per-site=4"); - true - } else { - false - } - } else if let Some(path) = &builder.config.rust_profile_use { - if compiler.stage == 1 { - cargo.rustflag(&format!("-Cprofile-use={path}")); - cargo.rustflag("-Cllvm-args=-pgo-warn-missing-function"); - true - } else { - false - } - } else { - false - }; - if is_collecting { - // Ensure paths to Rust sources are relative, not absolute. - cargo.rustflag(&format!( - "-Cllvm-args=-static-func-strip-dirname-prefix={}", - builder.config.src.components().count() - )); - } - - // We currently don't support cross-crate LTO in stage0. This also isn't hugely necessary - // and may just be a time sink. - if compiler.stage != 0 { - match builder.config.rust_lto { - RustcLto::Thin | RustcLto::Fat => { - // Since using LTO for optimizing dylibs is currently experimental, - // we need to pass -Zdylib-lto. - cargo.rustflag("-Zdylib-lto"); - // Cargo by default passes `-Cembed-bitcode=no` and doesn't pass `-Clto` when - // compiling dylibs (and their dependencies), even when LTO is enabled for the - // crate. Therefore, we need to override `-Clto` and `-Cembed-bitcode` here. - let lto_type = match builder.config.rust_lto { - RustcLto::Thin => "thin", - RustcLto::Fat => "fat", - _ => unreachable!(), - }; - cargo.rustflag(&format!("-Clto={lto_type}")); - cargo.rustflag("-Cembed-bitcode=yes"); - } - RustcLto::ThinLocal => { /* Do nothing, this is the default */ } - RustcLto::Off => { - cargo.rustflag("-Clto=off"); - } - } - } else if builder.config.rust_lto == RustcLto::Off { - cargo.rustflag("-Clto=off"); - } - - for krate in &*self.crates { - cargo.arg("-p").arg(krate); - } - - let _guard = builder.msg_sysroot_tool( - Kind::Build, - compiler.stage, - format_args!("compiler artifacts{}", crate_description(&self.crates)), - compiler.host, - target, - ); - let stamp = librustc_stamp(builder, compiler, target); - run_cargo( - builder, - cargo, - vec![], - &stamp, - vec![], - false, - true, // Only ship rustc_driver.so and .rmeta files, not all intermediate .rlib files. - ); - - // When building `librustc_driver.so` (like `libLLVM.so`) on linux, it can contain - // unexpected debuginfo from dependencies, for example from the C++ standard library used in - // our LLVM wrapper. Unless we're explicitly requesting `librustc_driver` to be built with - // debuginfo (via the debuginfo level of the executables using it): strip this debuginfo - // away after the fact. - if builder.config.rust_debuginfo_level_rustc == DebuginfoLevel::None - && builder.config.rust_debuginfo_level_tools == DebuginfoLevel::None - { - let target_root_dir = stamp.parent().unwrap(); - let rustc_driver = target_root_dir.join("librustc_driver.so"); - strip_debug(builder, target, &rustc_driver); - } - - builder.ensure(RustcLink::from_rustc( - self, - builder.compiler(compiler.stage, builder.config.build), - )); - } -} - -pub fn rustc_cargo(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelection, stage: u32) { - cargo - .arg("--features") - .arg(builder.rustc_features(builder.kind)) - .arg("--manifest-path") - .arg(builder.src.join("compiler/rustc/Cargo.toml")); - - cargo.rustdocflag("-Zcrate-attr=warn(rust_2018_idioms)"); - - rustc_cargo_env(builder, cargo, target, stage); -} - -pub fn rustc_cargo_env( - builder: &Builder<'_>, - cargo: &mut Cargo, - target: TargetSelection, - stage: u32, -) { - // Set some configuration variables picked up by build scripts and - // the compiler alike - cargo - .env("CFG_RELEASE", builder.rust_release()) - .env("CFG_RELEASE_CHANNEL", &builder.config.channel) - .env("CFG_VERSION", builder.rust_version()); - - if let Some(backend) = builder.config.default_codegen_backend() { - cargo.env("CFG_DEFAULT_CODEGEN_BACKEND", backend); - } - - let libdir_relative = builder.config.libdir_relative().unwrap_or_else(|| Path::new("lib")); - let target_config = builder.config.target_config.get(&target); - - cargo.env("CFG_LIBDIR_RELATIVE", libdir_relative); - - if let Some(ref ver_date) = builder.rust_info().commit_date() { - cargo.env("CFG_VER_DATE", ver_date); - } - if let Some(ref ver_hash) = builder.rust_info().sha() { - cargo.env("CFG_VER_HASH", ver_hash); - } - if !builder.unstable_features() { - cargo.env("CFG_DISABLE_UNSTABLE_FEATURES", "1"); - } - - // Prefer the current target's own default_linker, else a globally - // specified one. - if let Some(s) = target_config.and_then(|c| c.default_linker.as_ref()) { - cargo.env("CFG_DEFAULT_LINKER", s); - } else if let Some(ref s) = builder.config.rustc_default_linker { - cargo.env("CFG_DEFAULT_LINKER", s); - } - - if builder.config.rustc_parallel { - // keep in sync with `bootstrap/lib.rs:Build::rustc_features` - // `cfg` option for rustc, `features` option for cargo, for conditional compilation - cargo.rustflag("--cfg=parallel_compiler"); - cargo.rustdocflag("--cfg=parallel_compiler"); - } - if builder.config.rust_verify_llvm_ir { - cargo.env("RUSTC_VERIFY_LLVM_IR", "1"); - } - - // Note that this is disabled if LLVM itself is disabled or we're in a check - // build. If we are in a check build we still go ahead here presuming we've - // detected that LLVM is already built and good to go which helps prevent - // busting caches (e.g. like #71152). - if builder.config.llvm_enabled() { - let building_is_expensive = crate::llvm::prebuilt_llvm_config(builder, target).is_err(); - // `top_stage == stage` might be false for `check --stage 1`, if we are building the stage 1 compiler - let can_skip_build = builder.kind == Kind::Check && builder.top_stage == stage; - let should_skip_build = building_is_expensive && can_skip_build; - if !should_skip_build { - rustc_llvm_env(builder, cargo, target) - } - } -} - -/// Pass down configuration from the LLVM build into the build of -/// rustc_llvm and rustc_codegen_llvm. -fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelection) { - let target_config = builder.config.target_config.get(&target); - - if builder.is_rust_llvm(target) { - cargo.env("LLVM_RUSTLLVM", "1"); - } - let llvm::LlvmResult { llvm_config, .. } = builder.ensure(llvm::Llvm { target }); - cargo.env("LLVM_CONFIG", &llvm_config); - if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) { - cargo.env("CFG_LLVM_ROOT", s); - } - - // Some LLVM linker flags (-L and -l) may be needed to link `rustc_llvm`. Its build script - // expects these to be passed via the `LLVM_LINKER_FLAGS` env variable, separated by - // whitespace. - // - // For example: - // - on windows, when `clang-cl` is used with instrumentation, we need to manually add - // clang's runtime library resource directory so that the profiler runtime library can be - // found. This is to avoid the linker errors about undefined references to - // `__llvm_profile_instrument_memop` when linking `rustc_driver`. - let mut llvm_linker_flags = String::new(); - if builder.config.llvm_profile_generate && target.contains("msvc") { - if let Some(ref clang_cl_path) = builder.config.llvm_clang_cl { - // Add clang's runtime library directory to the search path - let clang_rt_dir = get_clang_cl_resource_dir(clang_cl_path); - llvm_linker_flags.push_str(&format!("-L{}", clang_rt_dir.display())); - } - } - - // The config can also specify its own llvm linker flags. - if let Some(ref s) = builder.config.llvm_ldflags { - if !llvm_linker_flags.is_empty() { - llvm_linker_flags.push_str(" "); - } - llvm_linker_flags.push_str(s); - } - - // Set the linker flags via the env var that `rustc_llvm`'s build script will read. - if !llvm_linker_flags.is_empty() { - cargo.env("LLVM_LINKER_FLAGS", llvm_linker_flags); - } - - // Building with a static libstdc++ is only supported on linux right now, - // not for MSVC or macOS - if builder.config.llvm_static_stdcpp - && !target.contains("freebsd") - && !target.contains("msvc") - && !target.contains("apple") - && !target.contains("solaris") - { - let file = compiler_file( - builder, - &builder.cxx(target).unwrap(), - target, - CLang::Cxx, - "libstdc++.a", - ); - cargo.env("LLVM_STATIC_STDCPP", file); - } - if builder.llvm_link_shared() { - cargo.env("LLVM_LINK_SHARED", "1"); - } - if builder.config.llvm_use_libcxx { - cargo.env("LLVM_USE_LIBCXX", "1"); - } - if builder.config.llvm_optimize && !builder.config.llvm_release_debuginfo { - cargo.env("LLVM_NDEBUG", "1"); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -struct RustcLink { - pub compiler: Compiler, - pub target_compiler: Compiler, - pub target: TargetSelection, - /// Not actually used; only present to make sure the cache invalidation is correct. - crates: Interned>, -} - -impl RustcLink { - fn from_rustc(rustc: Rustc, host_compiler: Compiler) -> Self { - Self { - compiler: host_compiler, - target_compiler: rustc.compiler, - target: rustc.target, - crates: rustc.crates, - } - } -} - -impl Step for RustcLink { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Same as `std_link`, only for librustc - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let target_compiler = self.target_compiler; - let target = self.target; - add_to_sysroot( - builder, - &builder.sysroot_libdir(target_compiler, target), - &builder.sysroot_libdir(target_compiler, compiler.host), - &librustc_stamp(builder, compiler, target), - ); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct CodegenBackend { - pub target: TargetSelection, - pub compiler: Compiler, - pub backend: Interned, -} - -fn needs_codegen_config(run: &RunConfig<'_>) -> bool { - let mut needs_codegen_cfg = false; - for path_set in &run.paths { - needs_codegen_cfg = match path_set { - PathSet::Set(set) => set.iter().any(|p| is_codegen_cfg_needed(p, run)), - PathSet::Suite(suite) => is_codegen_cfg_needed(&suite, run), - } - } - needs_codegen_cfg -} - -pub(crate) const CODEGEN_BACKEND_PREFIX: &str = "rustc_codegen_"; - -fn is_codegen_cfg_needed(path: &TaskPath, run: &RunConfig<'_>) -> bool { - if path.path.to_str().unwrap().contains(&CODEGEN_BACKEND_PREFIX) { - let mut needs_codegen_backend_config = true; - for &backend in &run.builder.config.rust_codegen_backends { - if path - .path - .to_str() - .unwrap() - .ends_with(&(CODEGEN_BACKEND_PREFIX.to_owned() + &backend)) - { - needs_codegen_backend_config = false; - } - } - if needs_codegen_backend_config { - run.builder.info( - "Warning: no codegen-backends config matched the requested path to build a codegen backend. \ - Help: add backend to codegen-backends in config.toml.", - ); - return true; - } - } - - return false; -} - -impl Step for CodegenBackend { - type Output = (); - const ONLY_HOSTS: bool = true; - // Only the backends specified in the `codegen-backends` entry of `config.toml` are built. - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.paths(&["compiler/rustc_codegen_cranelift", "compiler/rustc_codegen_gcc"]) - } - - fn make_run(run: RunConfig<'_>) { - if needs_codegen_config(&run) { - return; - } - - for &backend in &run.builder.config.rust_codegen_backends { - if backend == "llvm" { - continue; // Already built as part of rustc - } - - run.builder.ensure(CodegenBackend { - target: run.target, - compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), - backend, - }); - } - } - - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let target = self.target; - let backend = self.backend; - - builder.ensure(Rustc::new(compiler, target)); - - if builder.config.keep_stage.contains(&compiler.stage) { - builder.info( - "Warning: Using a potentially old codegen backend. \ - This may not behave well.", - ); - // Codegen backends are linked separately from this step today, so we don't do - // anything here. - return; - } - - let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); - if compiler_to_use != compiler { - builder.ensure(CodegenBackend { compiler: compiler_to_use, target, backend }); - return; - } - - let out_dir = builder.cargo_out(compiler, Mode::Codegen, target); - - let mut cargo = builder.cargo(compiler, Mode::Codegen, SourceType::InTree, target, "build"); - cargo - .arg("--manifest-path") - .arg(builder.src.join(format!("compiler/rustc_codegen_{backend}/Cargo.toml"))); - rustc_cargo_env(builder, &mut cargo, target, compiler.stage); - - let tmp_stamp = out_dir.join(".tmp.stamp"); - - let _guard = builder.msg_build(compiler, format_args!("codegen backend {backend}"), target); - let files = run_cargo(builder, cargo, vec![], &tmp_stamp, vec![], false, false); - if builder.config.dry_run() { - return; - } - let mut files = files.into_iter().filter(|f| { - let filename = f.file_name().unwrap().to_str().unwrap(); - is_dylib(filename) && filename.contains("rustc_codegen_") - }); - let codegen_backend = match files.next() { - Some(f) => f, - None => panic!("no dylibs built for codegen backend?"), - }; - if let Some(f) = files.next() { - panic!( - "codegen backend built two dylibs:\n{}\n{}", - codegen_backend.display(), - f.display() - ); - } - let stamp = codegen_backend_stamp(builder, compiler, target, backend); - let codegen_backend = codegen_backend.to_str().unwrap(); - t!(fs::write(&stamp, &codegen_backend)); - } -} - -/// Creates the `codegen-backends` folder for a compiler that's about to be -/// assembled as a complete compiler. -/// -/// This will take the codegen artifacts produced by `compiler` and link them -/// into an appropriate location for `target_compiler` to be a functional -/// compiler. -fn copy_codegen_backends_to_sysroot( - builder: &Builder<'_>, - compiler: Compiler, - target_compiler: Compiler, -) { - let target = target_compiler.host; - - // Note that this step is different than all the other `*Link` steps in - // that it's not assembling a bunch of libraries but rather is primarily - // moving the codegen backend into place. The codegen backend of rustc is - // not linked into the main compiler by default but is rather dynamically - // selected at runtime for inclusion. - // - // Here we're looking for the output dylib of the `CodegenBackend` step and - // we're copying that into the `codegen-backends` folder. - let dst = builder.sysroot_codegen_backends(target_compiler); - t!(fs::create_dir_all(&dst), dst); - - if builder.config.dry_run() { - return; - } - - for backend in builder.config.rust_codegen_backends.iter() { - if backend == "llvm" { - continue; // Already built as part of rustc - } - - let stamp = codegen_backend_stamp(builder, compiler, target, *backend); - let dylib = t!(fs::read_to_string(&stamp)); - let file = Path::new(&dylib); - let filename = file.file_name().unwrap().to_str().unwrap(); - // change `librustc_codegen_cranelift-xxxxxx.so` to - // `librustc_codegen_cranelift-release.so` - let target_filename = { - let dash = filename.find('-').unwrap(); - let dot = filename.find('.').unwrap(); - format!("{}-{}{}", &filename[..dash], builder.rust_release(), &filename[dot..]) - }; - builder.copy(&file, &dst.join(target_filename)); - } -} - -/// Cargo's output path for the standard library in a given stage, compiled -/// by a particular compiler for the specified target. -pub fn libstd_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { - builder.cargo_out(compiler, Mode::Std, target).join(".libstd.stamp") -} - -/// Cargo's output path for librustc in a given stage, compiled by a particular -/// compiler for the specified target. -pub fn librustc_stamp( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, -) -> PathBuf { - builder.cargo_out(compiler, Mode::Rustc, target).join(".librustc.stamp") -} - -/// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular -/// compiler for the specified target and backend. -fn codegen_backend_stamp( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, - backend: Interned, -) -> PathBuf { - builder - .cargo_out(compiler, Mode::Codegen, target) - .join(format!(".librustc_codegen_{backend}.stamp")) -} - -pub fn compiler_file( - builder: &Builder<'_>, - compiler: &Path, - target: TargetSelection, - c: CLang, - file: &str, -) -> PathBuf { - if builder.config.dry_run() { - return PathBuf::new(); - } - let mut cmd = Command::new(compiler); - cmd.args(builder.cflags(target, GitRepo::Rustc, c)); - cmd.arg(format!("-print-file-name={file}")); - let out = output(&mut cmd); - PathBuf::from(out.trim()) -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Sysroot { - pub compiler: Compiler, - /// See [`Std::force_recompile`]. - force_recompile: bool, -} - -impl Sysroot { - pub(crate) fn new(compiler: Compiler) -> Self { - Sysroot { compiler, force_recompile: false } - } -} - -impl Step for Sysroot { - type Output = Interned; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Returns the sysroot for the `compiler` specified that *this build system - /// generates*. - /// - /// That is, the sysroot for the stage0 compiler is not what the compiler - /// thinks it is by default, but it's the same as the default for stages - /// 1-3. - fn run(self, builder: &Builder<'_>) -> Interned { - let compiler = self.compiler; - let host_dir = builder.out.join(&compiler.host.triple); - - let sysroot_dir = |stage| { - if stage == 0 { - host_dir.join("stage0-sysroot") - } else if self.force_recompile && stage == compiler.stage { - host_dir.join(format!("stage{stage}-test-sysroot")) - } else if builder.download_rustc() && compiler.stage != builder.top_stage { - host_dir.join("ci-rustc-sysroot") - } else { - host_dir.join(format!("stage{}", stage)) - } - }; - let sysroot = sysroot_dir(compiler.stage); - - builder.verbose(&format!("Removing sysroot {} to avoid caching bugs", sysroot.display())); - let _ = fs::remove_dir_all(&sysroot); - t!(fs::create_dir_all(&sysroot)); - - // In some cases(see https://github.com/rust-lang/rust/issues/109314), when the stage0 - // compiler relies on more recent version of LLVM than the beta compiler, it may not - // be able to locate the correct LLVM in the sysroot. This situation typically occurs - // when we upgrade LLVM version while the beta compiler continues to use an older version. - // - // Make sure to add the correct version of LLVM into the stage0 sysroot. - if compiler.stage == 0 { - dist::maybe_install_llvm_target(builder, compiler.host, &sysroot); - } - - // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. - if builder.download_rustc() && compiler.stage != 0 { - assert_eq!( - builder.config.build, compiler.host, - "Cross-compiling is not yet supported with `download-rustc`", - ); - - // #102002, cleanup old toolchain folders when using download-rustc so people don't use them by accident. - for stage in 0..=2 { - if stage != compiler.stage { - let dir = sysroot_dir(stage); - if !dir.ends_with("ci-rustc-sysroot") { - let _ = fs::remove_dir_all(dir); - } - } - } - - // Copy the compiler into the correct sysroot. - // NOTE(#108767): We intentionally don't copy `rustc-dev` artifacts until they're requested with `builder.ensure(Rustc)`. - // This fixes an issue where we'd have multiple copies of libc in the sysroot with no way to tell which to load. - // There are a few quirks of bootstrap that interact to make this reliable: - // 1. The order `Step`s are run is hard-coded in `builder.rs` and not configurable. This - // avoids e.g. reordering `test::UiFulldeps` before `test::Ui` and causing the latter to - // fail because of duplicate metadata. - // 2. The sysroot is deleted and recreated between each invocation, so running `x test - // ui-fulldeps && x test ui` can't cause failures. - let mut filtered_files = Vec::new(); - let mut add_filtered_files = |suffix, contents| { - for path in contents { - let path = Path::new(&path); - if path.parent().map_or(false, |parent| parent.ends_with(&suffix)) { - filtered_files.push(path.file_name().unwrap().to_owned()); - } - } - }; - let suffix = format!("lib/rustlib/{}/lib", compiler.host); - add_filtered_files(suffix.as_str(), builder.config.ci_rustc_dev_contents()); - // NOTE: we can't copy std eagerly because `stage2-test-sysroot` needs to have only the - // newly compiled std, not the downloaded std. - add_filtered_files("lib", builder.config.ci_rust_std_contents()); - - let filtered_extensions = [ - OsStr::new("rmeta"), - OsStr::new("rlib"), - // FIXME: this is wrong when compiler.host != build, but we don't support that today - OsStr::new(std::env::consts::DLL_EXTENSION), - ]; - let ci_rustc_dir = builder.config.ci_rustc_dir(); - builder.cp_filtered(&ci_rustc_dir, &sysroot, &|path| { - if path.extension().map_or(true, |ext| !filtered_extensions.contains(&ext)) { - return true; - } - if !path.parent().map_or(true, |p| p.ends_with(&suffix)) { - return true; - } - if !filtered_files.iter().all(|f| f != path.file_name().unwrap()) { - builder.verbose_than(1, &format!("ignoring {}", path.display())); - false - } else { - true - } - }); - } - - // Symlink the source root into the same location inside the sysroot, - // where `rust-src` component would go (`$sysroot/lib/rustlib/src/rust`), - // so that any tools relying on `rust-src` also work for local builds, - // and also for translating the virtual `/rustc/$hash` back to the real - // directory (for running tests with `rust.remap-debuginfo = true`). - let sysroot_lib_rustlib_src = sysroot.join("lib/rustlib/src"); - t!(fs::create_dir_all(&sysroot_lib_rustlib_src)); - let sysroot_lib_rustlib_src_rust = sysroot_lib_rustlib_src.join("rust"); - if let Err(e) = symlink_dir(&builder.config, &builder.src, &sysroot_lib_rustlib_src_rust) { - eprintln!( - "warning: creating symbolic link `{}` to `{}` failed with {}", - sysroot_lib_rustlib_src_rust.display(), - builder.src.display(), - e, - ); - if builder.config.rust_remap_debuginfo { - eprintln!( - "warning: some `tests/ui` tests will fail when lacking `{}`", - sysroot_lib_rustlib_src_rust.display(), - ); - } - } - // Same for the rustc-src component. - let sysroot_lib_rustlib_rustcsrc = sysroot.join("lib/rustlib/rustc-src"); - t!(fs::create_dir_all(&sysroot_lib_rustlib_rustcsrc)); - let sysroot_lib_rustlib_rustcsrc_rust = sysroot_lib_rustlib_rustcsrc.join("rust"); - if let Err(e) = - symlink_dir(&builder.config, &builder.src, &sysroot_lib_rustlib_rustcsrc_rust) - { - eprintln!( - "warning: creating symbolic link `{}` to `{}` failed with {}", - sysroot_lib_rustlib_rustcsrc_rust.display(), - builder.src.display(), - e, - ); - } - - INTERNER.intern_path(sysroot) - } -} - -#[derive(Debug, Copy, PartialOrd, Ord, Clone, PartialEq, Eq, Hash)] -pub struct Assemble { - /// The compiler which we will produce in this step. Assemble itself will - /// take care of ensuring that the necessary prerequisites to do so exist, - /// that is, this target can be a stage2 compiler and Assemble will build - /// previous stages for you. - pub target_compiler: Compiler, -} - -impl Step for Assemble { - type Output = Compiler; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("compiler/rustc").path("compiler") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Assemble { - target_compiler: run.builder.compiler(run.builder.top_stage + 1, run.target), - }); - } - - /// Prepare a new compiler from the artifacts in `stage` - /// - /// This will assemble a compiler in `build/$host/stage$stage`. The compiler - /// must have been previously produced by the `stage - 1` builder.build - /// compiler. - fn run(self, builder: &Builder<'_>) -> Compiler { - let target_compiler = self.target_compiler; - - if target_compiler.stage == 0 { - assert_eq!( - builder.config.build, target_compiler.host, - "Cannot obtain compiler for non-native build triple at stage 0" - ); - // The stage 0 compiler for the build triple is always pre-built. - return target_compiler; - } - - // Get the compiler that we'll use to bootstrap ourselves. - // - // Note that this is where the recursive nature of the bootstrap - // happens, as this will request the previous stage's compiler on - // downwards to stage 0. - // - // Also note that we're building a compiler for the host platform. We - // only assume that we can run `build` artifacts, which means that to - // produce some other architecture compiler we need to start from - // `build` to get there. - // - // FIXME: It may be faster if we build just a stage 1 compiler and then - // use that to bootstrap this compiler forward. - let build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build); - - // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. - if builder.download_rustc() { - let sysroot = - builder.ensure(Sysroot { compiler: target_compiler, force_recompile: false }); - // Ensure that `libLLVM.so` ends up in the newly created target directory, - // so that tools using `rustc_private` can use it. - dist::maybe_install_llvm_target(builder, target_compiler.host, &sysroot); - // Lower stages use `ci-rustc-sysroot`, not stageN - if target_compiler.stage == builder.top_stage { - builder.info(&format!("Creating a sysroot for stage{stage} compiler (use `rustup toolchain link 'name' build/host/stage{stage}`)", stage=target_compiler.stage)); - } - return target_compiler; - } - - // Build the libraries for this compiler to link to (i.e., the libraries - // it uses at runtime). NOTE: Crates the target compiler compiles don't - // link to these. (FIXME: Is that correct? It seems to be correct most - // of the time but I think we do link to these for stage2/bin compilers - // when not performing a full bootstrap). - builder.ensure(Rustc::new(build_compiler, target_compiler.host)); - - // FIXME: For now patch over problems noted in #90244 by early returning here, even though - // we've not properly assembled the target sysroot. A full fix is pending further investigation, - // for now full bootstrap usage is rare enough that this is OK. - if target_compiler.stage >= 3 && !builder.config.full_bootstrap { - return target_compiler; - } - - for &backend in builder.config.rust_codegen_backends.iter() { - if backend == "llvm" { - continue; // Already built as part of rustc - } - - builder.ensure(CodegenBackend { - compiler: build_compiler, - target: target_compiler.host, - backend, - }); - } - - let lld_install = if builder.config.lld_enabled { - Some(builder.ensure(llvm::Lld { target: target_compiler.host })) - } else { - None - }; - - let stage = target_compiler.stage; - let host = target_compiler.host; - let (host_info, dir_name) = if build_compiler.host == host { - ("".into(), "host".into()) - } else { - (format!(" ({host})"), host.to_string()) - }; - // NOTE: "Creating a sysroot" is somewhat inconsistent with our internal terminology, since - // sysroots can temporarily be empty until we put the compiler inside. However, - // `ensure(Sysroot)` isn't really something that's user facing, so there shouldn't be any - // ambiguity. - let msg = format!( - "Creating a sysroot for stage{stage} compiler{host_info} (use `rustup toolchain link 'name' build/{dir_name}/stage{stage}`)" - ); - builder.info(&msg); - - // Link in all dylibs to the libdir - let stamp = librustc_stamp(builder, build_compiler, target_compiler.host); - let proc_macros = builder - .read_stamp_file(&stamp) - .into_iter() - .filter_map(|(path, dependency_type)| { - if dependency_type == DependencyType::Host { - Some(path.file_name().unwrap().to_owned().into_string().unwrap()) - } else { - None - } - }) - .collect::>(); - - let sysroot = builder.sysroot(target_compiler); - let rustc_libdir = builder.rustc_libdir(target_compiler); - t!(fs::create_dir_all(&rustc_libdir)); - let src_libdir = builder.sysroot_libdir(build_compiler, host); - for f in builder.read_dir(&src_libdir) { - let filename = f.file_name().into_string().unwrap(); - if (is_dylib(&filename) || is_debug_info(&filename)) && !proc_macros.contains(&filename) - { - builder.copy(&f.path(), &rustc_libdir.join(&filename)); - } - } - - copy_codegen_backends_to_sysroot(builder, build_compiler, target_compiler); - - // We prepend this bin directory to the user PATH when linking Rust binaries. To - // avoid shadowing the system LLD we rename the LLD we provide to `rust-lld`. - let libdir = builder.sysroot_libdir(target_compiler, target_compiler.host); - let libdir_bin = libdir.parent().unwrap().join("bin"); - t!(fs::create_dir_all(&libdir_bin)); - if let Some(lld_install) = lld_install { - let src_exe = exe("lld", target_compiler.host); - let dst_exe = exe("rust-lld", target_compiler.host); - builder.copy(&lld_install.join("bin").join(&src_exe), &libdir_bin.join(&dst_exe)); - // for `-Z gcc-ld=lld` - let gcc_ld_dir = libdir_bin.join("gcc-ld"); - t!(fs::create_dir(&gcc_ld_dir)); - let lld_wrapper_exe = builder.ensure(crate::tool::LldWrapper { - compiler: build_compiler, - target: target_compiler.host, - }); - for name in crate::LLD_FILE_NAMES { - builder.copy(&lld_wrapper_exe, &gcc_ld_dir.join(exe(name, target_compiler.host))); - } - } - - if builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) { - let llvm::LlvmResult { llvm_config, .. } = - builder.ensure(llvm::Llvm { target: target_compiler.host }); - if !builder.config.dry_run() { - let llvm_bin_dir = output(Command::new(llvm_config).arg("--bindir")); - let llvm_bin_dir = Path::new(llvm_bin_dir.trim()); - - // Since we've already built the LLVM tools, install them to the sysroot. - // This is the equivalent of installing the `llvm-tools-preview` component via - // rustup, and lets developers use a locally built toolchain to - // build projects that expect llvm tools to be present in the sysroot - // (e.g. the `bootimage` crate). - for tool in LLVM_TOOLS { - let tool_exe = exe(tool, target_compiler.host); - let src_path = llvm_bin_dir.join(&tool_exe); - // When using `download-ci-llvm`, some of the tools - // may not exist, so skip trying to copy them. - if src_path.exists() { - builder.copy(&src_path, &libdir_bin.join(&tool_exe)); - } - } - } - } - - // Ensure that `libLLVM.so` ends up in the newly build compiler directory, - // so that it can be found when the newly built `rustc` is run. - dist::maybe_install_llvm_runtime(builder, target_compiler.host, &sysroot); - dist::maybe_install_llvm_target(builder, target_compiler.host, &sysroot); - - // Link the compiler binary itself into place - let out_dir = builder.cargo_out(build_compiler, Mode::Rustc, host); - let rustc = out_dir.join(exe("rustc-main", host)); - let bindir = sysroot.join("bin"); - t!(fs::create_dir_all(&bindir)); - let compiler = builder.rustc(target_compiler); - builder.copy(&rustc, &compiler); - - target_compiler - } -} - -/// Link some files into a rustc sysroot. -/// -/// For a particular stage this will link the file listed in `stamp` into the -/// `sysroot_dst` provided. -pub fn add_to_sysroot( - builder: &Builder<'_>, - sysroot_dst: &Path, - sysroot_host_dst: &Path, - stamp: &Path, -) { - let self_contained_dst = &sysroot_dst.join("self-contained"); - t!(fs::create_dir_all(&sysroot_dst)); - t!(fs::create_dir_all(&sysroot_host_dst)); - t!(fs::create_dir_all(&self_contained_dst)); - for (path, dependency_type) in builder.read_stamp_file(stamp) { - let dst = match dependency_type { - DependencyType::Host => sysroot_host_dst, - DependencyType::Target => sysroot_dst, - DependencyType::TargetSelfContained => self_contained_dst, - }; - builder.copy(&path, &dst.join(path.file_name().unwrap())); - } -} - -pub fn run_cargo( - builder: &Builder<'_>, - cargo: Cargo, - tail_args: Vec, - stamp: &Path, - additional_target_deps: Vec<(PathBuf, DependencyType)>, - is_check: bool, - rlib_only_metadata: bool, -) -> Vec { - if builder.config.dry_run() { - return Vec::new(); - } - - // `target_root_dir` looks like $dir/$target/release - let target_root_dir = stamp.parent().unwrap(); - // `target_deps_dir` looks like $dir/$target/release/deps - let target_deps_dir = target_root_dir.join("deps"); - // `host_root_dir` looks like $dir/release - let host_root_dir = target_root_dir - .parent() - .unwrap() // chop off `release` - .parent() - .unwrap() // chop off `$target` - .join(target_root_dir.file_name().unwrap()); - - // Spawn Cargo slurping up its JSON output. We'll start building up the - // `deps` array of all files it generated along with a `toplevel` array of - // files we need to probe for later. - let mut deps = Vec::new(); - let mut toplevel = Vec::new(); - let ok = stream_cargo(builder, cargo, tail_args, &mut |msg| { - let (filenames, crate_types) = match msg { - CargoMessage::CompilerArtifact { - filenames, - target: CargoTarget { crate_types }, - .. - } => (filenames, crate_types), - _ => return, - }; - for filename in filenames { - // Skip files like executables - let mut keep = false; - if filename.ends_with(".lib") - || filename.ends_with(".a") - || is_debug_info(&filename) - || is_dylib(&filename) - { - // Always keep native libraries, rust dylibs and debuginfo - keep = true; - } - if is_check && filename.ends_with(".rmeta") { - // During check builds we need to keep crate metadata - keep = true; - } else if rlib_only_metadata { - if filename.contains("jemalloc_sys") - || filename.contains("rustc_smir") - || filename.contains("stable_mir") - { - // jemalloc_sys and rustc_smir are not linked into librustc_driver.so, - // so we need to distribute them as rlib to be able to use them. - keep |= filename.ends_with(".rlib"); - } else { - // Distribute the rest of the rustc crates as rmeta files only to reduce - // the tarball sizes by about 50%. The object files are linked into - // librustc_driver.so, so it is still possible to link against them. - keep |= filename.ends_with(".rmeta"); - } - } else { - // In all other cases keep all rlibs - keep |= filename.ends_with(".rlib"); - } - - if !keep { - continue; - } - - let filename = Path::new(&*filename); - - // If this was an output file in the "host dir" we don't actually - // worry about it, it's not relevant for us - if filename.starts_with(&host_root_dir) { - // Unless it's a proc macro used in the compiler - if crate_types.iter().any(|t| t == "proc-macro") { - deps.push((filename.to_path_buf(), DependencyType::Host)); - } - continue; - } - - // If this was output in the `deps` dir then this is a precise file - // name (hash included) so we start tracking it. - if filename.starts_with(&target_deps_dir) { - deps.push((filename.to_path_buf(), DependencyType::Target)); - continue; - } - - // Otherwise this was a "top level artifact" which right now doesn't - // have a hash in the name, but there's a version of this file in - // the `deps` folder which *does* have a hash in the name. That's - // the one we'll want to we'll probe for it later. - // - // We do not use `Path::file_stem` or `Path::extension` here, - // because some generated files may have multiple extensions e.g. - // `std-.dll.lib` on Windows. The aforementioned methods only - // split the file name by the last extension (`.lib`) while we need - // to split by all extensions (`.dll.lib`). - let expected_len = t!(filename.metadata()).len(); - let filename = filename.file_name().unwrap().to_str().unwrap(); - let mut parts = filename.splitn(2, '.'); - let file_stem = parts.next().unwrap().to_owned(); - let extension = parts.next().unwrap().to_owned(); - - toplevel.push((file_stem, extension, expected_len)); - } - }); - - if !ok { - crate::exit!(1); - } - - // Ok now we need to actually find all the files listed in `toplevel`. We've - // got a list of prefix/extensions and we basically just need to find the - // most recent file in the `deps` folder corresponding to each one. - let contents = t!(target_deps_dir.read_dir()) - .map(|e| t!(e)) - .map(|e| (e.path(), e.file_name().into_string().unwrap(), t!(e.metadata()))) - .collect::>(); - for (prefix, extension, expected_len) in toplevel { - let candidates = contents.iter().filter(|&&(_, ref filename, ref meta)| { - meta.len() == expected_len - && filename - .strip_prefix(&prefix[..]) - .map(|s| s.starts_with('-') && s.ends_with(&extension[..])) - .unwrap_or(false) - }); - let max = candidates.max_by_key(|&&(_, _, ref metadata)| { - metadata.modified().expect("mtime should be available on all relevant OSes") - }); - let path_to_add = match max { - Some(triple) => triple.0.to_str().unwrap(), - None => panic!("no output generated for {prefix:?} {extension:?}"), - }; - if is_dylib(path_to_add) { - let candidate = format!("{path_to_add}.lib"); - let candidate = PathBuf::from(candidate); - if candidate.exists() { - deps.push((candidate, DependencyType::Target)); - } - } - deps.push((path_to_add.into(), DependencyType::Target)); - } - - deps.extend(additional_target_deps); - deps.sort(); - let mut new_contents = Vec::new(); - for (dep, dependency_type) in deps.iter() { - new_contents.extend(match *dependency_type { - DependencyType::Host => b"h", - DependencyType::Target => b"t", - DependencyType::TargetSelfContained => b"s", - }); - new_contents.extend(dep.to_str().unwrap().as_bytes()); - new_contents.extend(b"\0"); - } - t!(fs::write(&stamp, &new_contents)); - deps.into_iter().map(|(d, _)| d).collect() -} - -pub fn stream_cargo( - builder: &Builder<'_>, - cargo: Cargo, - tail_args: Vec, - cb: &mut dyn FnMut(CargoMessage<'_>), -) -> bool { - let mut cargo = Command::from(cargo); - if builder.config.dry_run() { - return true; - } - // Instruct Cargo to give us json messages on stdout, critically leaving - // stderr as piped so we can get those pretty colors. - let mut message_format = if builder.config.json_output { - String::from("json") - } else { - String::from("json-render-diagnostics") - }; - if let Some(s) = &builder.config.rustc_error_format { - message_format.push_str(",json-diagnostic-"); - message_format.push_str(s); - } - cargo.arg("--message-format").arg(message_format).stdout(Stdio::piped()); - - for arg in tail_args { - cargo.arg(arg); - } - - builder.verbose(&format!("running: {cargo:?}")); - let mut child = match cargo.spawn() { - Ok(child) => child, - Err(e) => panic!("failed to execute command: {cargo:?}\nerror: {e}"), - }; - - // Spawn Cargo slurping up its JSON output. We'll start building up the - // `deps` array of all files it generated along with a `toplevel` array of - // files we need to probe for later. - let stdout = BufReader::new(child.stdout.take().unwrap()); - for line in stdout.lines() { - let line = t!(line); - match serde_json::from_str::>(&line) { - Ok(msg) => { - if builder.config.json_output { - // Forward JSON to stdout. - println!("{line}"); - } - cb(msg) - } - // If this was informational, just print it out and continue - Err(_) => println!("{line}"), - } - } - - // Make sure Cargo actually succeeded after we read all of its stdout. - let status = t!(child.wait()); - if builder.is_verbose() && !status.success() { - eprintln!( - "command did not execute successfully: {cargo:?}\n\ - expected success, got: {status}" - ); - } - status.success() -} - -#[derive(Deserialize)] -pub struct CargoTarget<'a> { - crate_types: Vec>, -} - -#[derive(Deserialize)] -#[serde(tag = "reason", rename_all = "kebab-case")] -pub enum CargoMessage<'a> { - CompilerArtifact { - package_id: Cow<'a, str>, - features: Vec>, - filenames: Vec>, - target: CargoTarget<'a>, - }, - BuildScriptExecuted { - package_id: Cow<'a, str>, - }, - BuildFinished { - success: bool, - }, -} - -pub fn strip_debug(builder: &Builder<'_>, target: TargetSelection, path: &Path) { - // FIXME: to make things simpler for now, limit this to the host and target where we know - // `strip -g` is both available and will fix the issue, i.e. on a x64 linux host that is not - // cross-compiling. Expand this to other appropriate targets in the future. - if target != "x86_64-unknown-linux-gnu" || target != builder.config.build || !path.exists() { - return; - } - - let previous_mtime = FileTime::from_last_modification_time(&path.metadata().unwrap()); - // Note: `output` will propagate any errors here. - output(Command::new("strip").arg("--strip-debug").arg(path)); - - // After running `strip`, we have to set the file modification time to what it was before, - // otherwise we risk Cargo invalidating its fingerprint and rebuilding the world next time - // bootstrap is invoked. - // - // An example of this is if we run this on librustc_driver.so. In the first invocation: - // - Cargo will build librustc_driver.so (mtime of 1) - // - Cargo will build rustc-main (mtime of 2) - // - Bootstrap will strip librustc_driver.so (changing the mtime to 3). - // - // In the second invocation of bootstrap, Cargo will see that the mtime of librustc_driver.so - // is greater than the mtime of rustc-main, and will rebuild rustc-main. That will then cause - // everything else (standard library, future stages...) to be rebuilt. - t!(filetime::set_file_mtime(path, previous_mtime)); -} diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs deleted file mode 100644 index 836328f94..000000000 --- a/src/bootstrap/config.rs +++ /dev/null @@ -1,2104 +0,0 @@ -//! Serialized configuration of a build. -//! -//! This module implements parsing `config.toml` configuration files to tweak -//! how the build runs. - -#[cfg(test)] -mod tests; - -use std::cell::{Cell, RefCell}; -use std::cmp; -use std::collections::{HashMap, HashSet}; -use std::env; -use std::fmt::{self, Display}; -use std::fs; -use std::io::IsTerminal; -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::str::FromStr; - -use crate::cache::{Interned, INTERNER}; -use crate::cc_detect::{ndk_compiler, Language}; -use crate::channel::{self, GitInfo}; -use crate::compile::CODEGEN_BACKEND_PREFIX; -pub use crate::flags::Subcommand; -use crate::flags::{Color, Flags, Warnings}; -use crate::util::{exe, output, t}; -use build_helper::exit; -use once_cell::sync::OnceCell; -use semver::Version; -use serde::{Deserialize, Deserializer}; -use serde_derive::Deserialize; - -macro_rules! check_ci_llvm { - ($name:expr) => { - assert!( - $name.is_none(), - "setting {} is incompatible with download-ci-llvm.", - stringify!($name) - ); - }; -} - -#[derive(Clone, Default)] -pub enum DryRun { - /// This isn't a dry run. - #[default] - Disabled, - /// This is a dry run enabled by bootstrap itself, so it can verify that no work is done. - SelfCheck, - /// This is a dry run enabled by the `--dry-run` flag. - UserSelected, -} - -#[derive(Copy, Clone, Default, PartialEq, Eq)] -pub enum DebuginfoLevel { - #[default] - None, - LineTablesOnly, - Limited, - Full, -} - -// NOTE: can't derive(Deserialize) because the intermediate trip through toml::Value only -// deserializes i64, and derive() only generates visit_u64 -impl<'de> Deserialize<'de> for DebuginfoLevel { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - use serde::de::Error; - - Ok(match Deserialize::deserialize(deserializer)? { - StringOrInt::String("none") | StringOrInt::Int(0) => DebuginfoLevel::None, - StringOrInt::String("line-tables-only") => DebuginfoLevel::LineTablesOnly, - StringOrInt::String("limited") | StringOrInt::Int(1) => DebuginfoLevel::Limited, - StringOrInt::String("full") | StringOrInt::Int(2) => DebuginfoLevel::Full, - StringOrInt::Int(n) => { - let other = serde::de::Unexpected::Signed(n); - return Err(D::Error::invalid_value(other, &"expected 0, 1, or 2")); - } - StringOrInt::String(s) => { - let other = serde::de::Unexpected::Str(s); - return Err(D::Error::invalid_value( - other, - &"expected none, line-tables-only, limited, or full", - )); - } - }) - } -} - -/// Suitable for passing to `-C debuginfo` -impl Display for DebuginfoLevel { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - use DebuginfoLevel::*; - f.write_str(match self { - None => "0", - LineTablesOnly => "line-tables-only", - Limited => "1", - Full => "2", - }) - } -} - -/// Global configuration for the entire build and/or bootstrap. -/// -/// This structure is parsed from `config.toml`, and some of the fields are inferred from `git` or build-time parameters. -/// -/// Note that this structure is not decoded directly into, but rather it is -/// filled out from the decoded forms of the structs below. For documentation -/// each field, see the corresponding fields in -/// `config.example.toml`. -#[derive(Default, Clone)] -pub struct Config { - pub changelog_seen: Option, - pub ccache: Option, - /// Call Build::ninja() instead of this. - pub ninja_in_file: bool, - pub verbose: usize, - pub submodules: Option, - pub compiler_docs: bool, - pub library_docs_private_items: bool, - pub docs_minification: bool, - pub docs: bool, - pub locked_deps: bool, - pub vendor: bool, - pub target_config: HashMap, - pub full_bootstrap: bool, - pub extended: bool, - pub tools: Option>, - pub sanitizers: bool, - pub profiler: bool, - pub omit_git_hash: bool, - pub skip: Vec, - pub include_default_paths: bool, - pub rustc_error_format: Option, - pub json_output: bool, - pub test_compare_mode: bool, - pub color: Color, - pub patch_binaries_for_nix: Option, - pub stage0_metadata: Stage0Metadata, - - pub stdout_is_tty: bool, - pub stderr_is_tty: bool, - - pub on_fail: Option, - pub stage: u32, - pub keep_stage: Vec, - pub keep_stage_std: Vec, - pub src: PathBuf, - /// defaults to `config.toml` - pub config: Option, - pub jobs: Option, - pub cmd: Subcommand, - pub incremental: bool, - pub dry_run: DryRun, - /// Arguments appearing after `--` to be forwarded to tools, - /// e.g. `--fix-broken` or test arguments. - pub free_args: Vec, - - /// `None` if we shouldn't download CI compiler artifacts, or the commit to download if we should. - #[cfg(not(test))] - download_rustc_commit: Option, - #[cfg(test)] - pub download_rustc_commit: Option, - - pub deny_warnings: bool, - pub backtrace_on_ice: bool, - - // llvm codegen options - pub llvm_assertions: bool, - pub llvm_tests: bool, - pub llvm_plugins: bool, - pub llvm_optimize: bool, - pub llvm_thin_lto: bool, - pub llvm_release_debuginfo: bool, - pub llvm_static_stdcpp: bool, - /// `None` if `llvm_from_ci` is true and we haven't yet downloaded llvm. - #[cfg(not(test))] - llvm_link_shared: Cell>, - #[cfg(test)] - pub llvm_link_shared: Cell>, - pub llvm_clang_cl: Option, - pub llvm_targets: Option, - pub llvm_experimental_targets: Option, - pub llvm_link_jobs: Option, - pub llvm_version_suffix: Option, - pub llvm_use_linker: Option, - pub llvm_allow_old_toolchain: bool, - pub llvm_polly: bool, - pub llvm_clang: bool, - pub llvm_enable_warnings: bool, - pub llvm_from_ci: bool, - pub llvm_build_config: HashMap, - - pub use_lld: bool, - pub lld_enabled: bool, - pub llvm_tools_enabled: bool, - - pub llvm_cflags: Option, - pub llvm_cxxflags: Option, - pub llvm_ldflags: Option, - pub llvm_use_libcxx: bool, - - // rust codegen options - pub rust_optimize: RustOptimize, - pub rust_codegen_units: Option, - pub rust_codegen_units_std: Option, - pub rust_debug_assertions: bool, - pub rust_debug_assertions_std: bool, - pub rust_overflow_checks: bool, - pub rust_overflow_checks_std: bool, - pub rust_debug_logging: bool, - pub rust_debuginfo_level_rustc: DebuginfoLevel, - pub rust_debuginfo_level_std: DebuginfoLevel, - pub rust_debuginfo_level_tools: DebuginfoLevel, - pub rust_debuginfo_level_tests: DebuginfoLevel, - pub rust_split_debuginfo: SplitDebuginfo, - pub rust_rpath: bool, - pub rustc_parallel: bool, - pub rustc_default_linker: Option, - pub rust_optimize_tests: bool, - pub rust_dist_src: bool, - pub rust_codegen_backends: Vec>, - pub rust_verify_llvm_ir: bool, - pub rust_thin_lto_import_instr_limit: Option, - pub rust_remap_debuginfo: bool, - pub rust_new_symbol_mangling: Option, - pub rust_profile_use: Option, - pub rust_profile_generate: Option, - pub rust_lto: RustcLto, - pub rust_validate_mir_opts: Option, - pub llvm_profile_use: Option, - pub llvm_profile_generate: bool, - pub llvm_libunwind_default: Option, - - pub reproducible_artifacts: Vec, - - pub build: TargetSelection, - pub hosts: Vec, - pub targets: Vec, - pub local_rebuild: bool, - pub jemalloc: bool, - pub control_flow_guard: bool, - - // dist misc - pub dist_sign_folder: Option, - pub dist_upload_addr: Option, - pub dist_compression_formats: Option>, - pub dist_compression_profile: String, - pub dist_include_mingw_linker: bool, - - // libstd features - pub backtrace: bool, // support for RUST_BACKTRACE - - // misc - pub low_priority: bool, - pub channel: String, - pub description: Option, - pub verbose_tests: bool, - pub save_toolstates: Option, - pub print_step_timings: bool, - pub print_step_rusage: bool, - pub missing_tools: bool, - - // Fallback musl-root for all targets - pub musl_root: Option, - pub prefix: Option, - pub sysconfdir: Option, - pub datadir: Option, - pub docdir: Option, - pub bindir: PathBuf, - pub libdir: Option, - pub mandir: Option, - pub codegen_tests: bool, - pub nodejs: Option, - pub npm: Option, - pub gdb: Option, - pub python: Option, - pub reuse: Option, - pub cargo_native_static: bool, - pub configure_args: Vec, - pub out: PathBuf, - pub rust_info: channel::GitInfo, - - // These are either the stage0 downloaded binaries or the locally installed ones. - pub initial_cargo: PathBuf, - pub initial_rustc: PathBuf, - - #[cfg(not(test))] - initial_rustfmt: RefCell, - #[cfg(test)] - pub initial_rustfmt: RefCell, - - pub paths: Vec, -} - -#[derive(Default, Deserialize, Clone)] -pub struct Stage0Metadata { - pub compiler: CompilerMetadata, - pub config: Stage0Config, - pub checksums_sha256: HashMap, - pub rustfmt: Option, -} -#[derive(Default, Deserialize, Clone)] -pub struct CompilerMetadata { - pub date: String, - pub version: String, -} - -#[derive(Default, Deserialize, Clone)] -pub struct Stage0Config { - pub dist_server: String, - pub artifacts_server: String, - pub artifacts_with_llvm_assertions_server: String, - pub git_merge_commit_email: String, - pub nightly_branch: String, -} -#[derive(Default, Deserialize, Clone)] -pub struct RustfmtMetadata { - pub date: String, - pub version: String, -} - -#[derive(Clone, Debug, Default)] -pub enum RustfmtState { - SystemToolchain(PathBuf), - Downloaded(PathBuf), - Unavailable, - #[default] - LazyEvaluated, -} - -#[derive(Debug, Default, Clone, Copy, PartialEq)] -pub enum LlvmLibunwind { - #[default] - No, - InTree, - System, -} - -impl FromStr for LlvmLibunwind { - type Err = String; - - fn from_str(value: &str) -> Result { - match value { - "no" => Ok(Self::No), - "in-tree" => Ok(Self::InTree), - "system" => Ok(Self::System), - invalid => Err(format!("Invalid value '{invalid}' for rust.llvm-libunwind config.")), - } - } -} - -#[derive(Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum SplitDebuginfo { - Packed, - Unpacked, - #[default] - Off, -} - -impl std::str::FromStr for SplitDebuginfo { - type Err = (); - - fn from_str(s: &str) -> Result { - match s { - "packed" => Ok(SplitDebuginfo::Packed), - "unpacked" => Ok(SplitDebuginfo::Unpacked), - "off" => Ok(SplitDebuginfo::Off), - _ => Err(()), - } - } -} - -impl SplitDebuginfo { - /// Returns the default `-Csplit-debuginfo` value for the current target. See the comment for - /// `rust.split-debuginfo` in `config.example.toml`. - fn default_for_platform(target: &str) -> Self { - if target.contains("apple") { - SplitDebuginfo::Unpacked - } else if target.contains("windows") { - SplitDebuginfo::Packed - } else { - SplitDebuginfo::Off - } - } -} - -/// LTO mode used for compiling rustc itself. -#[derive(Default, Clone, PartialEq, Debug)] -pub enum RustcLto { - Off, - #[default] - ThinLocal, - Thin, - Fat, -} - -impl std::str::FromStr for RustcLto { - type Err = String; - - fn from_str(s: &str) -> Result { - match s { - "thin-local" => Ok(RustcLto::ThinLocal), - "thin" => Ok(RustcLto::Thin), - "fat" => Ok(RustcLto::Fat), - "off" => Ok(RustcLto::Off), - _ => Err(format!("Invalid value for rustc LTO: {s}")), - } - } -} - -#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct TargetSelection { - pub triple: Interned, - file: Option>, - synthetic: bool, -} - -/// Newtype over `Vec` so we can implement custom parsing logic -#[derive(Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] -pub struct TargetSelectionList(Vec); - -pub fn target_selection_list(s: &str) -> Result { - Ok(TargetSelectionList( - s.split(",").filter(|s| !s.is_empty()).map(TargetSelection::from_user).collect(), - )) -} - -impl TargetSelection { - pub fn from_user(selection: &str) -> Self { - let path = Path::new(selection); - - let (triple, file) = if path.exists() { - let triple = path - .file_stem() - .expect("Target specification file has no file stem") - .to_str() - .expect("Target specification file stem is not UTF-8"); - - (triple, Some(selection)) - } else { - (selection, None) - }; - - let triple = INTERNER.intern_str(triple); - let file = file.map(|f| INTERNER.intern_str(f)); - - Self { triple, file, synthetic: false } - } - - pub fn create_synthetic(triple: &str, file: &str) -> Self { - Self { - triple: INTERNER.intern_str(triple), - file: Some(INTERNER.intern_str(file)), - synthetic: true, - } - } - - pub fn rustc_target_arg(&self) -> &str { - self.file.as_ref().unwrap_or(&self.triple) - } - - pub fn contains(&self, needle: &str) -> bool { - self.triple.contains(needle) - } - - pub fn starts_with(&self, needle: &str) -> bool { - self.triple.starts_with(needle) - } - - pub fn ends_with(&self, needle: &str) -> bool { - self.triple.ends_with(needle) - } - - // See src/bootstrap/synthetic_targets.rs - pub fn is_synthetic(&self) -> bool { - self.synthetic - } -} - -impl fmt::Display for TargetSelection { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.triple)?; - if let Some(file) = self.file { - write!(f, "({file})")?; - } - Ok(()) - } -} - -impl fmt::Debug for TargetSelection { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{self}") - } -} - -impl PartialEq<&str> for TargetSelection { - fn eq(&self, other: &&str) -> bool { - self.triple == *other - } -} - -/// Per-target configuration stored in the global configuration structure. -#[derive(Default, Clone)] -pub struct Target { - /// Some(path to llvm-config) if using an external LLVM. - pub llvm_config: Option, - pub llvm_has_rust_patches: Option, - /// Some(path to FileCheck) if one was specified. - pub llvm_filecheck: Option, - pub llvm_libunwind: Option, - pub cc: Option, - pub cxx: Option, - pub ar: Option, - pub ranlib: Option, - pub default_linker: Option, - pub linker: Option, - pub ndk: Option, - pub sanitizers: Option, - pub profiler: Option, - pub rpath: Option, - pub crt_static: Option, - pub musl_root: Option, - pub musl_libdir: Option, - pub wasi_root: Option, - pub qemu_rootfs: Option, - pub no_std: bool, -} - -impl Target { - pub fn from_triple(triple: &str) -> Self { - let mut target: Self = Default::default(); - if triple.contains("-none") || triple.contains("nvptx") || triple.contains("switch") { - target.no_std = true; - } - target - } -} -/// Structure of the `config.toml` file that configuration is read from. -/// -/// This structure uses `Decodable` to automatically decode a TOML configuration -/// file into this format, and then this is traversed and written into the above -/// `Config` structure. -#[derive(Deserialize, Default)] -#[serde(deny_unknown_fields, rename_all = "kebab-case")] -struct TomlConfig { - changelog_seen: Option, - build: Option, - install: Option, - llvm: Option, - rust: Option, - target: Option>, - dist: Option, - profile: Option, -} - -/// Describes how to handle conflicts in merging two [`TomlConfig`] -#[derive(Copy, Clone, Debug)] -enum ReplaceOpt { - /// Silently ignore a duplicated value - IgnoreDuplicate, - /// Override the current value, even if it's `Some` - Override, - /// Exit with an error on duplicate values - ErrorOnDuplicate, -} - -trait Merge { - fn merge(&mut self, other: Self, replace: ReplaceOpt); -} - -impl Merge for TomlConfig { - fn merge( - &mut self, - TomlConfig { build, install, llvm, rust, dist, target, profile: _, changelog_seen }: Self, - replace: ReplaceOpt, - ) { - fn do_merge(x: &mut Option, y: Option, replace: ReplaceOpt) { - if let Some(new) = y { - if let Some(original) = x { - original.merge(new, replace); - } else { - *x = Some(new); - } - } - } - self.changelog_seen.merge(changelog_seen, replace); - do_merge(&mut self.build, build, replace); - do_merge(&mut self.install, install, replace); - do_merge(&mut self.llvm, llvm, replace); - do_merge(&mut self.rust, rust, replace); - do_merge(&mut self.dist, dist, replace); - assert!(target.is_none(), "merging target-specific config is not currently supported"); - } -} - -// We are using a decl macro instead of a derive proc macro here to reduce the compile time of -// rustbuild. -macro_rules! define_config { - ($(#[$attr:meta])* struct $name:ident { - $($field:ident: Option<$field_ty:ty> = $field_key:literal,)* - }) => { - $(#[$attr])* - struct $name { - $($field: Option<$field_ty>,)* - } - - impl Merge for $name { - fn merge(&mut self, other: Self, replace: ReplaceOpt) { - $( - match replace { - ReplaceOpt::IgnoreDuplicate => { - if self.$field.is_none() { - self.$field = other.$field; - } - }, - ReplaceOpt::Override => { - if other.$field.is_some() { - self.$field = other.$field; - } - } - ReplaceOpt::ErrorOnDuplicate => { - if other.$field.is_some() { - if self.$field.is_some() { - if cfg!(test) { - panic!("overriding existing option") - } else { - eprintln!("overriding existing option: `{}`", stringify!($field)); - exit!(2); - } - } else { - self.$field = other.$field; - } - } - } - } - )* - } - } - - // The following is a trimmed version of what serde_derive generates. All parts not relevant - // for toml deserialization have been removed. This reduces the binary size and improves - // compile time of rustbuild. - impl<'de> Deserialize<'de> for $name { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - struct Field; - impl<'de> serde::de::Visitor<'de> for Field { - type Value = $name; - fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(concat!("struct ", stringify!($name))) - } - - #[inline] - fn visit_map(self, mut map: A) -> Result - where - A: serde::de::MapAccess<'de>, - { - $(let mut $field: Option<$field_ty> = None;)* - while let Some(key) = - match serde::de::MapAccess::next_key::(&mut map) { - Ok(val) => val, - Err(err) => { - return Err(err); - } - } - { - match &*key { - $($field_key => { - if $field.is_some() { - return Err(::duplicate_field( - $field_key, - )); - } - $field = match serde::de::MapAccess::next_value::<$field_ty>( - &mut map, - ) { - Ok(val) => Some(val), - Err(err) => { - return Err(err); - } - }; - })* - key => { - return Err(serde::de::Error::unknown_field(key, FIELDS)); - } - } - } - Ok($name { $($field),* }) - } - } - const FIELDS: &'static [&'static str] = &[ - $($field_key,)* - ]; - Deserializer::deserialize_struct( - deserializer, - stringify!($name), - FIELDS, - Field, - ) - } - } - } -} - -impl Merge for Option { - fn merge(&mut self, other: Self, replace: ReplaceOpt) { - match replace { - ReplaceOpt::IgnoreDuplicate => { - if self.is_none() { - *self = other; - } - } - ReplaceOpt::Override => { - if other.is_some() { - *self = other; - } - } - ReplaceOpt::ErrorOnDuplicate => { - if other.is_some() { - if self.is_some() { - if cfg!(test) { - panic!("overriding existing option") - } else { - eprintln!("overriding existing option"); - exit!(2); - } - } else { - *self = other; - } - } - } - } - } -} - -define_config! { - /// TOML representation of various global build decisions. - #[derive(Default)] - struct Build { - build: Option = "build", - host: Option> = "host", - target: Option> = "target", - build_dir: Option = "build-dir", - cargo: Option = "cargo", - rustc: Option = "rustc", - rustfmt: Option = "rustfmt", - docs: Option = "docs", - compiler_docs: Option = "compiler-docs", - library_docs_private_items: Option = "library-docs-private-items", - docs_minification: Option = "docs-minification", - submodules: Option = "submodules", - gdb: Option = "gdb", - nodejs: Option = "nodejs", - npm: Option = "npm", - python: Option = "python", - reuse: Option = "reuse", - locked_deps: Option = "locked-deps", - vendor: Option = "vendor", - full_bootstrap: Option = "full-bootstrap", - extended: Option = "extended", - tools: Option> = "tools", - verbose: Option = "verbose", - sanitizers: Option = "sanitizers", - profiler: Option = "profiler", - cargo_native_static: Option = "cargo-native-static", - low_priority: Option = "low-priority", - configure_args: Option> = "configure-args", - local_rebuild: Option = "local-rebuild", - print_step_timings: Option = "print-step-timings", - print_step_rusage: Option = "print-step-rusage", - check_stage: Option = "check-stage", - doc_stage: Option = "doc-stage", - build_stage: Option = "build-stage", - test_stage: Option = "test-stage", - install_stage: Option = "install-stage", - dist_stage: Option = "dist-stage", - bench_stage: Option = "bench-stage", - patch_binaries_for_nix: Option = "patch-binaries-for-nix", - // NOTE: only parsed by bootstrap.py, `--feature build-metrics` enables metrics unconditionally - metrics: Option = "metrics", - } -} - -define_config! { - /// TOML representation of various global install decisions. - struct Install { - prefix: Option = "prefix", - sysconfdir: Option = "sysconfdir", - docdir: Option = "docdir", - bindir: Option = "bindir", - libdir: Option = "libdir", - mandir: Option = "mandir", - datadir: Option = "datadir", - } -} - -define_config! { - /// TOML representation of how the LLVM build is configured. - struct Llvm { - optimize: Option = "optimize", - thin_lto: Option = "thin-lto", - release_debuginfo: Option = "release-debuginfo", - assertions: Option = "assertions", - tests: Option = "tests", - plugins: Option = "plugins", - ccache: Option = "ccache", - static_libstdcpp: Option = "static-libstdcpp", - ninja: Option = "ninja", - targets: Option = "targets", - experimental_targets: Option = "experimental-targets", - link_jobs: Option = "link-jobs", - link_shared: Option = "link-shared", - version_suffix: Option = "version-suffix", - clang_cl: Option = "clang-cl", - cflags: Option = "cflags", - cxxflags: Option = "cxxflags", - ldflags: Option = "ldflags", - use_libcxx: Option = "use-libcxx", - use_linker: Option = "use-linker", - allow_old_toolchain: Option = "allow-old-toolchain", - polly: Option = "polly", - clang: Option = "clang", - enable_warnings: Option = "enable-warnings", - download_ci_llvm: Option = "download-ci-llvm", - build_config: Option> = "build-config", - } -} - -define_config! { - struct Dist { - sign_folder: Option = "sign-folder", - gpg_password_file: Option = "gpg-password-file", - upload_addr: Option = "upload-addr", - src_tarball: Option = "src-tarball", - missing_tools: Option = "missing-tools", - compression_formats: Option> = "compression-formats", - compression_profile: Option = "compression-profile", - include_mingw_linker: Option = "include-mingw-linker", - } -} - -#[derive(Clone, Debug, Deserialize)] -#[serde(untagged)] -pub enum StringOrBool { - String(String), - Bool(bool), -} - -impl Default for StringOrBool { - fn default() -> StringOrBool { - StringOrBool::Bool(false) - } -} - -impl StringOrBool { - fn is_string_or_true(&self) -> bool { - matches!(self, Self::String(_) | Self::Bool(true)) - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum RustOptimize { - String(String), - Int(u8), - Bool(bool), -} - -impl Default for RustOptimize { - fn default() -> RustOptimize { - RustOptimize::Bool(false) - } -} - -impl<'de> Deserialize<'de> for RustOptimize { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - deserializer.deserialize_any(OptimizeVisitor) - } -} - -struct OptimizeVisitor; - -impl<'de> serde::de::Visitor<'de> for OptimizeVisitor { - type Value = RustOptimize; - - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str(r#"one of: 0, 1, 2, 3, "s", "z", true, false"#) - } - - fn visit_str(self, value: &str) -> Result - where - E: serde::de::Error, - { - if ["s", "z"].iter().find(|x| **x == value).is_some() { - Ok(RustOptimize::String(value.to_string())) - } else { - Err(format_optimize_error_msg(value)).map_err(serde::de::Error::custom) - } - } - - fn visit_i64(self, value: i64) -> Result - where - E: serde::de::Error, - { - if matches!(value, 0..=3) { - Ok(RustOptimize::Int(value as u8)) - } else { - Err(format_optimize_error_msg(value)).map_err(serde::de::Error::custom) - } - } - - fn visit_bool(self, value: bool) -> Result - where - E: serde::de::Error, - { - Ok(RustOptimize::Bool(value)) - } -} - -fn format_optimize_error_msg(v: impl std::fmt::Display) -> String { - format!( - r#"unrecognized option for rust optimize: "{v}", expected one of 0, 1, 2, 3, "s", "z", true, false"# - ) -} - -impl RustOptimize { - pub(crate) fn is_release(&self) -> bool { - match &self { - RustOptimize::Bool(true) | RustOptimize::String(_) => true, - RustOptimize::Int(i) => *i > 0, - RustOptimize::Bool(false) => false, - } - } - - pub(crate) fn get_opt_level(&self) -> Option { - match &self { - RustOptimize::String(s) => Some(s.clone()), - RustOptimize::Int(i) => Some(i.to_string()), - RustOptimize::Bool(_) => None, - } - } -} - -#[derive(Deserialize)] -#[serde(untagged)] -enum StringOrInt<'a> { - String(&'a str), - Int(i64), -} -define_config! { - /// TOML representation of how the Rust build is configured. - struct Rust { - optimize: Option = "optimize", - debug: Option = "debug", - codegen_units: Option = "codegen-units", - codegen_units_std: Option = "codegen-units-std", - debug_assertions: Option = "debug-assertions", - debug_assertions_std: Option = "debug-assertions-std", - overflow_checks: Option = "overflow-checks", - overflow_checks_std: Option = "overflow-checks-std", - debug_logging: Option = "debug-logging", - debuginfo_level: Option = "debuginfo-level", - debuginfo_level_rustc: Option = "debuginfo-level-rustc", - debuginfo_level_std: Option = "debuginfo-level-std", - debuginfo_level_tools: Option = "debuginfo-level-tools", - debuginfo_level_tests: Option = "debuginfo-level-tests", - split_debuginfo: Option = "split-debuginfo", - run_dsymutil: Option = "run-dsymutil", - backtrace: Option = "backtrace", - incremental: Option = "incremental", - parallel_compiler: Option = "parallel-compiler", - default_linker: Option = "default-linker", - channel: Option = "channel", - description: Option = "description", - musl_root: Option = "musl-root", - rpath: Option = "rpath", - verbose_tests: Option = "verbose-tests", - optimize_tests: Option = "optimize-tests", - codegen_tests: Option = "codegen-tests", - omit_git_hash: Option = "omit-git-hash", - dist_src: Option = "dist-src", - save_toolstates: Option = "save-toolstates", - codegen_backends: Option> = "codegen-backends", - lld: Option = "lld", - use_lld: Option = "use-lld", - llvm_tools: Option = "llvm-tools", - deny_warnings: Option = "deny-warnings", - backtrace_on_ice: Option = "backtrace-on-ice", - verify_llvm_ir: Option = "verify-llvm-ir", - thin_lto_import_instr_limit: Option = "thin-lto-import-instr-limit", - remap_debuginfo: Option = "remap-debuginfo", - jemalloc: Option = "jemalloc", - test_compare_mode: Option = "test-compare-mode", - llvm_libunwind: Option = "llvm-libunwind", - control_flow_guard: Option = "control-flow-guard", - new_symbol_mangling: Option = "new-symbol-mangling", - profile_generate: Option = "profile-generate", - profile_use: Option = "profile-use", - // ignored; this is set from an env var set by bootstrap.py - download_rustc: Option = "download-rustc", - lto: Option = "lto", - validate_mir_opts: Option = "validate-mir-opts", - } -} - -define_config! { - /// TOML representation of how each build target is configured. - struct TomlTarget { - cc: Option = "cc", - cxx: Option = "cxx", - ar: Option = "ar", - ranlib: Option = "ranlib", - default_linker: Option = "default-linker", - linker: Option = "linker", - llvm_config: Option = "llvm-config", - llvm_has_rust_patches: Option = "llvm-has-rust-patches", - llvm_filecheck: Option = "llvm-filecheck", - llvm_libunwind: Option = "llvm-libunwind", - android_ndk: Option = "android-ndk", - sanitizers: Option = "sanitizers", - profiler: Option = "profiler", - rpath: Option = "rpath", - crt_static: Option = "crt-static", - musl_root: Option = "musl-root", - musl_libdir: Option = "musl-libdir", - wasi_root: Option = "wasi-root", - qemu_rootfs: Option = "qemu-rootfs", - no_std: Option = "no-std", - } -} - -impl Config { - pub fn default_opts() -> Config { - let mut config = Config::default(); - config.llvm_optimize = true; - config.ninja_in_file = true; - config.llvm_static_stdcpp = false; - config.backtrace = true; - config.rust_optimize = RustOptimize::Bool(true); - config.rust_optimize_tests = true; - config.submodules = None; - config.docs = true; - config.docs_minification = true; - config.rust_rpath = true; - config.channel = "dev".to_string(); - config.codegen_tests = true; - config.rust_dist_src = true; - config.rust_codegen_backends = vec![INTERNER.intern_str("llvm")]; - config.deny_warnings = true; - config.bindir = "bin".into(); - config.dist_include_mingw_linker = true; - config.dist_compression_profile = "fast".into(); - - config.stdout_is_tty = std::io::stdout().is_terminal(); - config.stderr_is_tty = std::io::stderr().is_terminal(); - - // set by build.rs - config.build = TargetSelection::from_user(&env!("BUILD_TRIPLE")); - - let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - // Undo `src/bootstrap` - config.src = manifest_dir.parent().unwrap().parent().unwrap().to_owned(); - config.out = PathBuf::from("build"); - - config - } - - pub fn parse(args: &[String]) -> Config { - #[cfg(test)] - fn get_toml(_: &Path) -> TomlConfig { - TomlConfig::default() - } - - #[cfg(not(test))] - fn get_toml(file: &Path) -> TomlConfig { - let contents = - t!(fs::read_to_string(file), format!("config file {} not found", file.display())); - // Deserialize to Value and then TomlConfig to prevent the Deserialize impl of - // TomlConfig and sub types to be monomorphized 5x by toml. - toml::from_str(&contents) - .and_then(|table: toml::Value| TomlConfig::deserialize(table)) - .unwrap_or_else(|err| { - eprintln!("failed to parse TOML configuration '{}': {err}", file.display()); - exit!(2); - }) - } - Self::parse_inner(args, get_toml) - } - - fn parse_inner(args: &[String], get_toml: impl Fn(&Path) -> TomlConfig) -> Config { - let mut flags = Flags::parse(&args); - let mut config = Config::default_opts(); - - // Set flags. - config.paths = std::mem::take(&mut flags.paths); - config.skip = flags.skip.into_iter().chain(flags.exclude).collect(); - config.include_default_paths = flags.include_default_paths; - config.rustc_error_format = flags.rustc_error_format; - config.json_output = flags.json_output; - config.on_fail = flags.on_fail; - config.jobs = Some(threads_from_config(flags.jobs as u32)); - config.cmd = flags.cmd; - config.incremental = flags.incremental; - config.dry_run = if flags.dry_run { DryRun::UserSelected } else { DryRun::Disabled }; - config.keep_stage = flags.keep_stage; - config.keep_stage_std = flags.keep_stage_std; - config.color = flags.color; - config.free_args = std::mem::take(&mut flags.free_args); - config.llvm_profile_use = flags.llvm_profile_use; - config.llvm_profile_generate = flags.llvm_profile_generate; - - // Infer the rest of the configuration. - - // Infer the source directory. This is non-trivial because we want to support a downloaded bootstrap binary, - // running on a completely machine from where it was compiled. - let mut cmd = Command::new("git"); - // NOTE: we cannot support running from outside the repository because the only path we have available - // is set at compile time, which can be wrong if bootstrap was downloaded from source. - // We still support running outside the repository if we find we aren't in a git directory. - cmd.arg("rev-parse").arg("--show-toplevel"); - // Discard stderr because we expect this to fail when building from a tarball. - let output = cmd - .stderr(std::process::Stdio::null()) - .output() - .ok() - .and_then(|output| if output.status.success() { Some(output) } else { None }); - if let Some(output) = output { - let git_root = String::from_utf8(output.stdout).unwrap(); - // We need to canonicalize this path to make sure it uses backslashes instead of forward slashes. - let git_root = PathBuf::from(git_root.trim()).canonicalize().unwrap(); - let s = git_root.to_str().unwrap(); - - // Bootstrap is quite bad at handling /? in front of paths - let src = match s.strip_prefix("\\\\?\\") { - Some(p) => PathBuf::from(p), - None => PathBuf::from(git_root), - }; - // If this doesn't have at least `stage0.json`, we guessed wrong. This can happen when, - // for example, the build directory is inside of another unrelated git directory. - // In that case keep the original `CARGO_MANIFEST_DIR` handling. - // - // NOTE: this implies that downloadable bootstrap isn't supported when the build directory is outside - // the source directory. We could fix that by setting a variable from all three of python, ./x, and x.ps1. - if src.join("src").join("stage0.json").exists() { - config.src = src; - } - } else { - // We're building from a tarball, not git sources. - // We don't support pre-downloaded bootstrap in this case. - } - - if cfg!(test) { - // Use the build directory of the original x.py invocation, so that we can set `initial_rustc` properly. - config.out = Path::new( - &env::var_os("CARGO_TARGET_DIR").expect("cargo test directly is not supported"), - ) - .parent() - .unwrap() - .to_path_buf(); - } - - let stage0_json = t!(std::fs::read(&config.src.join("src").join("stage0.json"))); - - config.stage0_metadata = t!(serde_json::from_slice::(&stage0_json)); - - // Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory. - let toml_path = flags - .config - .clone() - .or_else(|| env::var_os("RUST_BOOTSTRAP_CONFIG").map(PathBuf::from)); - let using_default_path = toml_path.is_none(); - let mut toml_path = toml_path.unwrap_or_else(|| PathBuf::from("config.toml")); - if using_default_path && !toml_path.exists() { - toml_path = config.src.join(toml_path); - } - - // Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path, - // but not if `config.toml` hasn't been created. - let mut toml = if !using_default_path || toml_path.exists() { - config.config = Some(toml_path.clone()); - get_toml(&toml_path) - } else { - config.config = None; - TomlConfig::default() - }; - - if let Some(include) = &toml.profile { - // Allows creating alias for profile names, allowing - // profiles to be renamed while maintaining back compatibility - // Keep in sync with `profile_aliases` in bootstrap.py - let profile_aliases = HashMap::from([("user", "dist")]); - let include = match profile_aliases.get(include.as_str()) { - Some(alias) => alias, - None => include.as_str(), - }; - let mut include_path = config.src.clone(); - include_path.push("src"); - include_path.push("bootstrap"); - include_path.push("defaults"); - include_path.push(format!("config.{include}.toml")); - let included_toml = get_toml(&include_path); - toml.merge(included_toml, ReplaceOpt::IgnoreDuplicate); - } - - let mut override_toml = TomlConfig::default(); - for option in flags.set.iter() { - fn get_table(option: &str) -> Result { - toml::from_str(&option) - .and_then(|table: toml::Value| TomlConfig::deserialize(table)) - } - - let mut err = match get_table(option) { - Ok(v) => { - override_toml.merge(v, ReplaceOpt::ErrorOnDuplicate); - continue; - } - Err(e) => e, - }; - // We want to be able to set string values without quotes, - // like in `configure.py`. Try adding quotes around the right hand side - if let Some((key, value)) = option.split_once("=") { - if !value.contains('"') { - match get_table(&format!(r#"{key}="{value}""#)) { - Ok(v) => { - override_toml.merge(v, ReplaceOpt::ErrorOnDuplicate); - continue; - } - Err(e) => err = e, - } - } - } - eprintln!("failed to parse override `{option}`: `{err}"); - exit!(2) - } - toml.merge(override_toml, ReplaceOpt::Override); - - config.changelog_seen = toml.changelog_seen; - - let build = toml.build.unwrap_or_default(); - if let Some(file_build) = build.build { - config.build = TargetSelection::from_user(&file_build); - }; - - set(&mut config.out, flags.build_dir.or_else(|| build.build_dir.map(PathBuf::from))); - // NOTE: Bootstrap spawns various commands with different working directories. - // To avoid writing to random places on the file system, `config.out` needs to be an absolute path. - if !config.out.is_absolute() { - // `canonicalize` requires the path to already exist. Use our vendored copy of `absolute` instead. - config.out = crate::util::absolute(&config.out); - } - - config.initial_rustc = if let Some(rustc) = build.rustc { - // FIXME(#115065): re-enable this check - // config.check_build_rustc_version(&rustc); - PathBuf::from(rustc) - } else { - config.download_beta_toolchain(); - config.out.join(config.build.triple).join("stage0/bin/rustc") - }; - - config.initial_cargo = build - .cargo - .map(|cargo| { - t!(PathBuf::from(cargo).canonicalize(), "`initial_cargo` not found on disk") - }) - .unwrap_or_else(|| config.out.join(config.build.triple).join("stage0/bin/cargo")); - - // NOTE: it's important this comes *after* we set `initial_rustc` just above. - if config.dry_run() { - let dir = config.out.join("tmp-dry-run"); - t!(fs::create_dir_all(&dir)); - config.out = dir; - } - - config.hosts = if let Some(TargetSelectionList(arg_host)) = flags.host { - arg_host - } else if let Some(file_host) = build.host { - file_host.iter().map(|h| TargetSelection::from_user(h)).collect() - } else { - vec![config.build] - }; - config.targets = if let Some(TargetSelectionList(arg_target)) = flags.target { - arg_target - } else if let Some(file_target) = build.target { - file_target.iter().map(|h| TargetSelection::from_user(h)).collect() - } else { - // If target is *not* configured, then default to the host - // toolchains. - config.hosts.clone() - }; - - config.nodejs = build.nodejs.map(PathBuf::from); - config.npm = build.npm.map(PathBuf::from); - config.gdb = build.gdb.map(PathBuf::from); - config.python = build.python.map(PathBuf::from); - config.reuse = build.reuse.map(PathBuf::from); - config.submodules = build.submodules; - set(&mut config.low_priority, build.low_priority); - set(&mut config.compiler_docs, build.compiler_docs); - set(&mut config.library_docs_private_items, build.library_docs_private_items); - set(&mut config.docs_minification, build.docs_minification); - set(&mut config.docs, build.docs); - set(&mut config.locked_deps, build.locked_deps); - set(&mut config.vendor, build.vendor); - set(&mut config.full_bootstrap, build.full_bootstrap); - set(&mut config.extended, build.extended); - config.tools = build.tools; - set(&mut config.verbose, build.verbose); - set(&mut config.sanitizers, build.sanitizers); - set(&mut config.profiler, build.profiler); - set(&mut config.cargo_native_static, build.cargo_native_static); - set(&mut config.configure_args, build.configure_args); - set(&mut config.local_rebuild, build.local_rebuild); - set(&mut config.print_step_timings, build.print_step_timings); - set(&mut config.print_step_rusage, build.print_step_rusage); - config.patch_binaries_for_nix = build.patch_binaries_for_nix; - - config.verbose = cmp::max(config.verbose, flags.verbose as usize); - - if let Some(install) = toml.install { - config.prefix = install.prefix.map(PathBuf::from); - config.sysconfdir = install.sysconfdir.map(PathBuf::from); - config.datadir = install.datadir.map(PathBuf::from); - config.docdir = install.docdir.map(PathBuf::from); - set(&mut config.bindir, install.bindir.map(PathBuf::from)); - config.libdir = install.libdir.map(PathBuf::from); - config.mandir = install.mandir.map(PathBuf::from); - } - - // Store off these values as options because if they're not provided - // we'll infer default values for them later - let mut llvm_assertions = None; - let mut llvm_tests = None; - let mut llvm_plugins = None; - let mut debug = None; - let mut debug_assertions = None; - let mut debug_assertions_std = None; - let mut overflow_checks = None; - let mut overflow_checks_std = None; - let mut debug_logging = None; - let mut debuginfo_level = None; - let mut debuginfo_level_rustc = None; - let mut debuginfo_level_std = None; - let mut debuginfo_level_tools = None; - let mut debuginfo_level_tests = None; - let mut optimize = None; - let mut omit_git_hash = None; - - if let Some(rust) = toml.rust { - set(&mut config.channel, rust.channel); - - config.download_rustc_commit = config.download_ci_rustc_commit(rust.download_rustc); - // This list is incomplete, please help by expanding it! - if config.download_rustc_commit.is_some() { - // We need the channel used by the downloaded compiler to match the one we set for rustdoc; - // otherwise rustdoc-ui tests break. - let ci_channel = t!(fs::read_to_string(config.src.join("src/ci/channel"))); - let ci_channel = ci_channel.trim_end(); - if config.channel != ci_channel - && !(config.channel == "dev" && ci_channel == "nightly") - { - panic!( - "setting rust.channel={} is incompatible with download-rustc", - config.channel - ); - } - } - - debug = rust.debug; - debug_assertions = rust.debug_assertions; - debug_assertions_std = rust.debug_assertions_std; - overflow_checks = rust.overflow_checks; - overflow_checks_std = rust.overflow_checks_std; - debug_logging = rust.debug_logging; - debuginfo_level = rust.debuginfo_level; - debuginfo_level_rustc = rust.debuginfo_level_rustc; - debuginfo_level_std = rust.debuginfo_level_std; - debuginfo_level_tools = rust.debuginfo_level_tools; - debuginfo_level_tests = rust.debuginfo_level_tests; - - config.rust_split_debuginfo = rust - .split_debuginfo - .as_deref() - .map(SplitDebuginfo::from_str) - .map(|v| v.expect("invalid value for rust.split_debuginfo")) - .unwrap_or(SplitDebuginfo::default_for_platform(&config.build.triple)); - optimize = rust.optimize; - omit_git_hash = rust.omit_git_hash; - config.rust_new_symbol_mangling = rust.new_symbol_mangling; - set(&mut config.rust_optimize_tests, rust.optimize_tests); - set(&mut config.codegen_tests, rust.codegen_tests); - set(&mut config.rust_rpath, rust.rpath); - set(&mut config.jemalloc, rust.jemalloc); - set(&mut config.test_compare_mode, rust.test_compare_mode); - set(&mut config.backtrace, rust.backtrace); - config.description = rust.description; - set(&mut config.rust_dist_src, rust.dist_src); - set(&mut config.verbose_tests, rust.verbose_tests); - // in the case "false" is set explicitly, do not overwrite the command line args - if let Some(true) = rust.incremental { - config.incremental = true; - } - set(&mut config.use_lld, rust.use_lld); - set(&mut config.lld_enabled, rust.lld); - set(&mut config.llvm_tools_enabled, rust.llvm_tools); - config.rustc_parallel = rust.parallel_compiler.unwrap_or(false); - config.rustc_default_linker = rust.default_linker; - config.musl_root = rust.musl_root.map(PathBuf::from); - config.save_toolstates = rust.save_toolstates.map(PathBuf::from); - set( - &mut config.deny_warnings, - match flags.warnings { - Warnings::Deny => Some(true), - Warnings::Warn => Some(false), - Warnings::Default => rust.deny_warnings, - }, - ); - set(&mut config.backtrace_on_ice, rust.backtrace_on_ice); - set(&mut config.rust_verify_llvm_ir, rust.verify_llvm_ir); - config.rust_thin_lto_import_instr_limit = rust.thin_lto_import_instr_limit; - set(&mut config.rust_remap_debuginfo, rust.remap_debuginfo); - set(&mut config.control_flow_guard, rust.control_flow_guard); - config.llvm_libunwind_default = rust - .llvm_libunwind - .map(|v| v.parse().expect("failed to parse rust.llvm-libunwind")); - - if let Some(ref backends) = rust.codegen_backends { - let available_backends = vec!["llvm", "cranelift", "gcc"]; - - config.rust_codegen_backends = backends.iter().map(|s| { - if let Some(backend) = s.strip_prefix(CODEGEN_BACKEND_PREFIX) { - if available_backends.contains(&backend) { - panic!("Invalid value '{s}' for 'rust.codegen-backends'. Instead, please use '{backend}'."); - } else { - println!("help: '{s}' for 'rust.codegen-backends' might fail. \ - Codegen backends are mostly defined without the '{CODEGEN_BACKEND_PREFIX}' prefix. \ - In this case, it would be referred to as '{backend}'."); - } - } - - INTERNER.intern_str(s) - }).collect(); - } - - config.rust_codegen_units = rust.codegen_units.map(threads_from_config); - config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config); - config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use); - config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate); - config.rust_lto = rust - .lto - .as_deref() - .map(|value| RustcLto::from_str(value).unwrap()) - .unwrap_or_default(); - config.rust_validate_mir_opts = rust.validate_mir_opts; - } else { - config.rust_profile_use = flags.rust_profile_use; - config.rust_profile_generate = flags.rust_profile_generate; - } - - config.reproducible_artifacts = flags.reproducible_artifact; - - // rust_info must be set before is_ci_llvm_available() is called. - let default = config.channel == "dev"; - config.omit_git_hash = omit_git_hash.unwrap_or(default); - config.rust_info = GitInfo::new(config.omit_git_hash, &config.src); - - if let Some(llvm) = toml.llvm { - match llvm.ccache { - Some(StringOrBool::String(ref s)) => config.ccache = Some(s.to_string()), - Some(StringOrBool::Bool(true)) => { - config.ccache = Some("ccache".to_string()); - } - Some(StringOrBool::Bool(false)) | None => {} - } - set(&mut config.ninja_in_file, llvm.ninja); - llvm_assertions = llvm.assertions; - llvm_tests = llvm.tests; - llvm_plugins = llvm.plugins; - set(&mut config.llvm_optimize, llvm.optimize); - set(&mut config.llvm_thin_lto, llvm.thin_lto); - set(&mut config.llvm_release_debuginfo, llvm.release_debuginfo); - set(&mut config.llvm_static_stdcpp, llvm.static_libstdcpp); - if let Some(v) = llvm.link_shared { - config.llvm_link_shared.set(Some(v)); - } - config.llvm_targets = llvm.targets.clone(); - config.llvm_experimental_targets = llvm.experimental_targets.clone(); - config.llvm_link_jobs = llvm.link_jobs; - config.llvm_version_suffix = llvm.version_suffix.clone(); - config.llvm_clang_cl = llvm.clang_cl.clone(); - - config.llvm_cflags = llvm.cflags.clone(); - config.llvm_cxxflags = llvm.cxxflags.clone(); - config.llvm_ldflags = llvm.ldflags.clone(); - set(&mut config.llvm_use_libcxx, llvm.use_libcxx); - config.llvm_use_linker = llvm.use_linker.clone(); - config.llvm_allow_old_toolchain = llvm.allow_old_toolchain.unwrap_or(false); - config.llvm_polly = llvm.polly.unwrap_or(false); - config.llvm_clang = llvm.clang.unwrap_or(false); - config.llvm_enable_warnings = llvm.enable_warnings.unwrap_or(false); - config.llvm_build_config = llvm.build_config.clone().unwrap_or(Default::default()); - - let asserts = llvm_assertions.unwrap_or(false); - config.llvm_from_ci = match llvm.download_ci_llvm { - Some(StringOrBool::String(s)) => { - assert_eq!(s, "if-available", "unknown option `{s}` for download-ci-llvm"); - crate::llvm::is_ci_llvm_available(&config, asserts) - } - Some(StringOrBool::Bool(b)) => b, - None => { - config.channel == "dev" && crate::llvm::is_ci_llvm_available(&config, asserts) - } - }; - - if config.llvm_from_ci { - // None of the LLVM options, except assertions, are supported - // when using downloaded LLVM. We could just ignore these but - // that's potentially confusing, so force them to not be - // explicitly set. The defaults and CI defaults don't - // necessarily match but forcing people to match (somewhat - // arbitrary) CI configuration locally seems bad/hard. - check_ci_llvm!(llvm.optimize); - check_ci_llvm!(llvm.thin_lto); - check_ci_llvm!(llvm.release_debuginfo); - // CI-built LLVM can be either dynamic or static. We won't know until we download it. - check_ci_llvm!(llvm.link_shared); - check_ci_llvm!(llvm.static_libstdcpp); - check_ci_llvm!(llvm.targets); - check_ci_llvm!(llvm.experimental_targets); - check_ci_llvm!(llvm.link_jobs); - check_ci_llvm!(llvm.clang_cl); - check_ci_llvm!(llvm.version_suffix); - check_ci_llvm!(llvm.cflags); - check_ci_llvm!(llvm.cxxflags); - check_ci_llvm!(llvm.ldflags); - check_ci_llvm!(llvm.use_libcxx); - check_ci_llvm!(llvm.use_linker); - check_ci_llvm!(llvm.allow_old_toolchain); - check_ci_llvm!(llvm.polly); - check_ci_llvm!(llvm.clang); - check_ci_llvm!(llvm.build_config); - check_ci_llvm!(llvm.plugins); - } - - // NOTE: can never be hit when downloading from CI, since we call `check_ci_llvm!(thin_lto)` above. - if config.llvm_thin_lto && llvm.link_shared.is_none() { - // If we're building with ThinLTO on, by default we want to link - // to LLVM shared, to avoid re-doing ThinLTO (which happens in - // the link step) with each stage. - config.llvm_link_shared.set(Some(true)); - } - } else { - config.llvm_from_ci = - config.channel == "dev" && crate::llvm::is_ci_llvm_available(&config, false); - } - - if let Some(t) = toml.target { - for (triple, cfg) in t { - let mut target = Target::from_triple(&triple); - - if let Some(ref s) = cfg.llvm_config { - if config.download_rustc_commit.is_some() && triple == &*config.build.triple { - panic!( - "setting llvm_config for the host is incompatible with download-rustc" - ); - } - target.llvm_config = Some(config.src.join(s)); - } - target.llvm_has_rust_patches = cfg.llvm_has_rust_patches; - if let Some(ref s) = cfg.llvm_filecheck { - target.llvm_filecheck = Some(config.src.join(s)); - } - target.llvm_libunwind = cfg - .llvm_libunwind - .as_ref() - .map(|v| v.parse().expect("failed to parse rust.llvm-libunwind")); - if let Some(ref s) = cfg.android_ndk { - target.ndk = Some(config.src.join(s)); - } - if let Some(s) = cfg.no_std { - target.no_std = s; - } - target.cc = cfg.cc.map(PathBuf::from).or_else(|| { - target.ndk.as_ref().map(|ndk| ndk_compiler(Language::C, &triple, ndk)) - }); - target.cxx = cfg.cxx.map(PathBuf::from).or_else(|| { - target.ndk.as_ref().map(|ndk| ndk_compiler(Language::CPlusPlus, &triple, ndk)) - }); - target.ar = cfg.ar.map(PathBuf::from); - target.ranlib = cfg.ranlib.map(PathBuf::from); - target.linker = cfg.linker.map(PathBuf::from); - target.crt_static = cfg.crt_static; - target.musl_root = cfg.musl_root.map(PathBuf::from); - target.musl_libdir = cfg.musl_libdir.map(PathBuf::from); - target.wasi_root = cfg.wasi_root.map(PathBuf::from); - target.qemu_rootfs = cfg.qemu_rootfs.map(PathBuf::from); - target.sanitizers = cfg.sanitizers; - target.profiler = cfg.profiler; - target.rpath = cfg.rpath; - - config.target_config.insert(TargetSelection::from_user(&triple), target); - } - } - - if config.llvm_from_ci { - let triple = &config.build.triple; - let ci_llvm_bin = config.ci_llvm_root().join("bin"); - let build_target = config - .target_config - .entry(config.build) - .or_insert_with(|| Target::from_triple(&triple)); - - check_ci_llvm!(build_target.llvm_config); - check_ci_llvm!(build_target.llvm_filecheck); - build_target.llvm_config = Some(ci_llvm_bin.join(exe("llvm-config", config.build))); - build_target.llvm_filecheck = Some(ci_llvm_bin.join(exe("FileCheck", config.build))); - } - - if let Some(t) = toml.dist { - config.dist_sign_folder = t.sign_folder.map(PathBuf::from); - config.dist_upload_addr = t.upload_addr; - config.dist_compression_formats = t.compression_formats; - set(&mut config.dist_compression_profile, t.compression_profile); - set(&mut config.rust_dist_src, t.src_tarball); - set(&mut config.missing_tools, t.missing_tools); - set(&mut config.dist_include_mingw_linker, t.include_mingw_linker) - } - - if let Some(r) = build.rustfmt { - *config.initial_rustfmt.borrow_mut() = if r.exists() { - RustfmtState::SystemToolchain(r) - } else { - RustfmtState::Unavailable - }; - } - - // Now that we've reached the end of our configuration, infer the - // default values for all options that we haven't otherwise stored yet. - - config.llvm_assertions = llvm_assertions.unwrap_or(false); - config.llvm_tests = llvm_tests.unwrap_or(false); - config.llvm_plugins = llvm_plugins.unwrap_or(false); - config.rust_optimize = optimize.unwrap_or(RustOptimize::Bool(true)); - - let default = debug == Some(true); - config.rust_debug_assertions = debug_assertions.unwrap_or(default); - config.rust_debug_assertions_std = - debug_assertions_std.unwrap_or(config.rust_debug_assertions); - config.rust_overflow_checks = overflow_checks.unwrap_or(default); - config.rust_overflow_checks_std = - overflow_checks_std.unwrap_or(config.rust_overflow_checks); - - config.rust_debug_logging = debug_logging.unwrap_or(config.rust_debug_assertions); - - let with_defaults = |debuginfo_level_specific: Option<_>| { - debuginfo_level_specific.or(debuginfo_level).unwrap_or(if debug == Some(true) { - DebuginfoLevel::Limited - } else { - DebuginfoLevel::None - }) - }; - config.rust_debuginfo_level_rustc = with_defaults(debuginfo_level_rustc); - config.rust_debuginfo_level_std = with_defaults(debuginfo_level_std); - config.rust_debuginfo_level_tools = with_defaults(debuginfo_level_tools); - config.rust_debuginfo_level_tests = debuginfo_level_tests.unwrap_or(DebuginfoLevel::None); - - let download_rustc = config.download_rustc_commit.is_some(); - // See https://github.com/rust-lang/compiler-team/issues/326 - config.stage = match config.cmd { - Subcommand::Check { .. } => flags.stage.or(build.check_stage).unwrap_or(0), - // `download-rustc` only has a speed-up for stage2 builds. Default to stage2 unless explicitly overridden. - Subcommand::Doc { .. } => { - flags.stage.or(build.doc_stage).unwrap_or(if download_rustc { 2 } else { 0 }) - } - Subcommand::Build { .. } => { - flags.stage.or(build.build_stage).unwrap_or(if download_rustc { 2 } else { 1 }) - } - Subcommand::Test { .. } => { - flags.stage.or(build.test_stage).unwrap_or(if download_rustc { 2 } else { 1 }) - } - Subcommand::Bench { .. } => flags.stage.or(build.bench_stage).unwrap_or(2), - Subcommand::Dist { .. } => flags.stage.or(build.dist_stage).unwrap_or(2), - Subcommand::Install { .. } => flags.stage.or(build.install_stage).unwrap_or(2), - // These are all bootstrap tools, which don't depend on the compiler. - // The stage we pass shouldn't matter, but use 0 just in case. - Subcommand::Clean { .. } - | Subcommand::Clippy { .. } - | Subcommand::Fix { .. } - | Subcommand::Run { .. } - | Subcommand::Setup { .. } - | Subcommand::Format { .. } - | Subcommand::Suggest { .. } => flags.stage.unwrap_or(0), - }; - - // CI should always run stage 2 builds, unless it specifically states otherwise - #[cfg(not(test))] - if flags.stage.is_none() && crate::CiEnv::current() != crate::CiEnv::None { - match config.cmd { - Subcommand::Test { .. } - | Subcommand::Doc { .. } - | Subcommand::Build { .. } - | Subcommand::Bench { .. } - | Subcommand::Dist { .. } - | Subcommand::Install { .. } => { - assert_eq!( - config.stage, 2, - "x.py should be run with `--stage 2` on CI, but was run with `--stage {}`", - config.stage, - ); - } - Subcommand::Clean { .. } - | Subcommand::Check { .. } - | Subcommand::Clippy { .. } - | Subcommand::Fix { .. } - | Subcommand::Run { .. } - | Subcommand::Setup { .. } - | Subcommand::Format { .. } - | Subcommand::Suggest { .. } => {} - } - } - - config - } - - pub(crate) fn dry_run(&self) -> bool { - match self.dry_run { - DryRun::Disabled => false, - DryRun::SelfCheck | DryRun::UserSelected => true, - } - } - - /// Runs a command, printing out nice contextual information if it fails. - /// Exits if the command failed to execute at all, otherwise returns its - /// `status.success()`. - #[deprecated = "use `Builder::try_run` instead where possible"] - pub(crate) fn try_run(&self, cmd: &mut Command) -> Result<(), ()> { - if self.dry_run() { - return Ok(()); - } - self.verbose(&format!("running: {cmd:?}")); - build_helper::util::try_run(cmd, self.is_verbose()) - } - - /// A git invocation which runs inside the source directory. - /// - /// Use this rather than `Command::new("git")` in order to support out-of-tree builds. - pub(crate) fn git(&self) -> Command { - let mut git = Command::new("git"); - git.current_dir(&self.src); - git - } - - pub(crate) fn test_args(&self) -> Vec<&str> { - let mut test_args = match self.cmd { - Subcommand::Test { ref test_args, .. } | Subcommand::Bench { ref test_args, .. } => { - test_args.iter().flat_map(|s| s.split_whitespace()).collect() - } - _ => vec![], - }; - test_args.extend(self.free_args.iter().map(|s| s.as_str())); - test_args - } - - pub(crate) fn args(&self) -> Vec<&str> { - let mut args = match self.cmd { - Subcommand::Run { ref args, .. } => { - args.iter().flat_map(|s| s.split_whitespace()).collect() - } - _ => vec![], - }; - args.extend(self.free_args.iter().map(|s| s.as_str())); - args - } - - /// Bootstrap embeds a version number into the name of shared libraries it uploads in CI. - /// Return the version it would have used for the given commit. - pub(crate) fn artifact_version_part(&self, commit: &str) -> String { - let (channel, version) = if self.rust_info.is_managed_git_subrepository() { - let mut channel = self.git(); - channel.arg("show").arg(format!("{commit}:src/ci/channel")); - let channel = output(&mut channel); - let mut version = self.git(); - version.arg("show").arg(format!("{commit}:src/version")); - let version = output(&mut version); - (channel.trim().to_owned(), version.trim().to_owned()) - } else { - let channel = fs::read_to_string(self.src.join("src/ci/channel")); - let version = fs::read_to_string(self.src.join("src/version")); - match (channel, version) { - (Ok(channel), Ok(version)) => { - (channel.trim().to_owned(), version.trim().to_owned()) - } - (channel, version) => { - let src = self.src.display(); - eprintln!("error: failed to determine artifact channel and/or version"); - eprintln!( - "help: consider using a git checkout or ensure these files are readable" - ); - if let Err(channel) = channel { - eprintln!("reading {src}/src/ci/channel failed: {channel:?}"); - } - if let Err(version) = version { - eprintln!("reading {src}/src/version failed: {version:?}"); - } - panic!(); - } - } - }; - - match channel.as_str() { - "stable" => version, - "beta" => channel, - "nightly" => channel, - other => unreachable!("{:?} is not recognized as a valid channel", other), - } - } - - /// Try to find the relative path of `bindir`, otherwise return it in full. - pub fn bindir_relative(&self) -> &Path { - let bindir = &self.bindir; - if bindir.is_absolute() { - // Try to make it relative to the prefix. - if let Some(prefix) = &self.prefix { - if let Ok(stripped) = bindir.strip_prefix(prefix) { - return stripped; - } - } - } - bindir - } - - /// Try to find the relative path of `libdir`. - pub fn libdir_relative(&self) -> Option<&Path> { - let libdir = self.libdir.as_ref()?; - if libdir.is_relative() { - Some(libdir) - } else { - // Try to make it relative to the prefix. - libdir.strip_prefix(self.prefix.as_ref()?).ok() - } - } - - /// The absolute path to the downloaded LLVM artifacts. - pub(crate) fn ci_llvm_root(&self) -> PathBuf { - assert!(self.llvm_from_ci); - self.out.join(&*self.build.triple).join("ci-llvm") - } - - /// Directory where the extracted `rustc-dev` component is stored. - pub(crate) fn ci_rustc_dir(&self) -> PathBuf { - assert!(self.download_rustc()); - self.out.join(self.build.triple).join("ci-rustc") - } - - /// Determine whether llvm should be linked dynamically. - /// - /// If `false`, llvm should be linked statically. - /// This is computed on demand since LLVM might have to first be downloaded from CI. - pub(crate) fn llvm_link_shared(&self) -> bool { - let mut opt = self.llvm_link_shared.get(); - if opt.is_none() && self.dry_run() { - // just assume static for now - dynamic linking isn't supported on all platforms - return false; - } - - let llvm_link_shared = *opt.get_or_insert_with(|| { - if self.llvm_from_ci { - self.maybe_download_ci_llvm(); - let ci_llvm = self.ci_llvm_root(); - let link_type = t!( - std::fs::read_to_string(ci_llvm.join("link-type.txt")), - format!("CI llvm missing: {}", ci_llvm.display()) - ); - link_type == "dynamic" - } else { - // unclear how thought-through this default is, but it maintains compatibility with - // previous behavior - false - } - }); - self.llvm_link_shared.set(opt); - llvm_link_shared - } - - /// Return whether we will use a downloaded, pre-compiled version of rustc, or just build from source. - pub(crate) fn download_rustc(&self) -> bool { - self.download_rustc_commit().is_some() - } - - pub(crate) fn download_rustc_commit(&self) -> Option<&str> { - static DOWNLOAD_RUSTC: OnceCell> = OnceCell::new(); - if self.dry_run() && DOWNLOAD_RUSTC.get().is_none() { - // avoid trying to actually download the commit - return self.download_rustc_commit.as_deref(); - } - - DOWNLOAD_RUSTC - .get_or_init(|| match &self.download_rustc_commit { - None => None, - Some(commit) => { - self.download_ci_rustc(commit); - Some(commit.clone()) - } - }) - .as_deref() - } - - pub(crate) fn initial_rustfmt(&self) -> Option { - match &mut *self.initial_rustfmt.borrow_mut() { - RustfmtState::SystemToolchain(p) | RustfmtState::Downloaded(p) => Some(p.clone()), - RustfmtState::Unavailable => None, - r @ RustfmtState::LazyEvaluated => { - if self.dry_run() { - return Some(PathBuf::new()); - } - let path = self.maybe_download_rustfmt(); - *r = if let Some(p) = &path { - RustfmtState::Downloaded(p.clone()) - } else { - RustfmtState::Unavailable - }; - path - } - } - } - - pub fn verbose(&self, msg: &str) { - if self.verbose > 0 { - println!("{msg}"); - } - } - - pub fn sanitizers_enabled(&self, target: TargetSelection) -> bool { - self.target_config.get(&target).map(|t| t.sanitizers).flatten().unwrap_or(self.sanitizers) - } - - pub fn any_sanitizers_enabled(&self) -> bool { - self.target_config.values().any(|t| t.sanitizers == Some(true)) || self.sanitizers - } - - pub fn profiler_path(&self, target: TargetSelection) -> Option<&str> { - match self.target_config.get(&target)?.profiler.as_ref()? { - StringOrBool::String(s) => Some(s), - StringOrBool::Bool(_) => None, - } - } - - pub fn profiler_enabled(&self, target: TargetSelection) -> bool { - self.target_config - .get(&target) - .and_then(|t| t.profiler.as_ref()) - .map(StringOrBool::is_string_or_true) - .unwrap_or(self.profiler) - } - - pub fn any_profiler_enabled(&self) -> bool { - self.target_config.values().any(|t| matches!(&t.profiler, Some(p) if p.is_string_or_true())) - || self.profiler - } - - pub fn rpath_enabled(&self, target: TargetSelection) -> bool { - self.target_config.get(&target).map(|t| t.rpath).flatten().unwrap_or(self.rust_rpath) - } - - pub fn llvm_enabled(&self) -> bool { - self.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) - } - - pub fn llvm_libunwind(&self, target: TargetSelection) -> LlvmLibunwind { - self.target_config - .get(&target) - .and_then(|t| t.llvm_libunwind) - .or(self.llvm_libunwind_default) - .unwrap_or(if target.contains("fuchsia") { - LlvmLibunwind::InTree - } else { - LlvmLibunwind::No - }) - } - - pub fn submodules(&self, rust_info: &GitInfo) -> bool { - self.submodules.unwrap_or(rust_info.is_managed_git_subrepository()) - } - - pub fn default_codegen_backend(&self) -> Option> { - self.rust_codegen_backends.get(0).cloned() - } - - pub fn check_build_rustc_version(&self, rustc_path: &str) { - if self.dry_run() { - return; - } - - // check rustc version is same or lower with 1 apart from the building one - let mut cmd = Command::new(rustc_path); - cmd.arg("--version"); - let rustc_output = output(&mut cmd) - .lines() - .next() - .unwrap() - .split(' ') - .nth(1) - .unwrap() - .split('-') - .next() - .unwrap() - .to_owned(); - let rustc_version = Version::parse(&rustc_output.trim()).unwrap(); - let source_version = - Version::parse(&fs::read_to_string(self.src.join("src/version")).unwrap().trim()) - .unwrap(); - if !(source_version == rustc_version - || (source_version.major == rustc_version.major - && (source_version.minor == rustc_version.minor - || source_version.minor == rustc_version.minor + 1))) - { - let prev_version = format!("{}.{}.x", source_version.major, source_version.minor - 1); - eprintln!( - "Unexpected rustc version: {rustc_version}, we should use {prev_version}/{source_version} to build source with {source_version}" - ); - exit!(1); - } - } - - /// Returns the commit to download, or `None` if we shouldn't download CI artifacts. - fn download_ci_rustc_commit(&self, download_rustc: Option) -> Option { - // If `download-rustc` is not set, default to rebuilding. - let if_unchanged = match download_rustc { - None | Some(StringOrBool::Bool(false)) => return None, - Some(StringOrBool::Bool(true)) => false, - Some(StringOrBool::String(s)) if s == "if-unchanged" => true, - Some(StringOrBool::String(other)) => { - panic!("unrecognized option for download-rustc: {other}") - } - }; - - // Handle running from a directory other than the top level - let top_level = output(self.git().args(&["rev-parse", "--show-toplevel"])); - let top_level = top_level.trim_end(); - let compiler = format!("{top_level}/compiler/"); - let library = format!("{top_level}/library/"); - - // Look for a version to compare to based on the current commit. - // Only commits merged by bors will have CI artifacts. - let merge_base = output( - self.git() - .arg("rev-list") - .arg(format!("--author={}", self.stage0_metadata.config.git_merge_commit_email)) - .args(&["-n1", "--first-parent", "HEAD"]), - ); - let commit = merge_base.trim_end(); - if commit.is_empty() { - println!("error: could not find commit hash for downloading rustc"); - println!("help: maybe your repository history is too shallow?"); - println!("help: consider disabling `download-rustc`"); - println!("help: or fetch enough history to include one upstream commit"); - crate::exit!(1); - } - - // Warn if there were changes to the compiler or standard library since the ancestor commit. - let has_changes = !t!(self - .git() - .args(&["diff-index", "--quiet", &commit, "--", &compiler, &library]) - .status()) - .success(); - if has_changes { - if if_unchanged { - if self.verbose > 0 { - println!( - "warning: saw changes to compiler/ or library/ since {commit}; \ - ignoring `download-rustc`" - ); - } - return None; - } - println!( - "warning: `download-rustc` is enabled, but there are changes to \ - compiler/ or library/" - ); - } - - Some(commit.to_string()) - } -} - -fn set(field: &mut T, val: Option) { - if let Some(v) = val { - *field = v; - } -} - -fn threads_from_config(v: u32) -> u32 { - match v { - 0 => std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32, - n => n, - } -} diff --git a/src/bootstrap/config/tests.rs b/src/bootstrap/config/tests.rs deleted file mode 100644 index aac76cdcb..000000000 --- a/src/bootstrap/config/tests.rs +++ /dev/null @@ -1,198 +0,0 @@ -use crate::config::TomlConfig; - -use super::{Config, Flags}; -use clap::CommandFactory; -use serde::Deserialize; -use std::{env, path::Path}; - -fn parse(config: &str) -> Config { - Config::parse_inner(&["check".to_owned(), "--config=/does/not/exist".to_owned()], |&_| { - toml::from_str(config).unwrap() - }) -} - -#[test] -fn download_ci_llvm() { - if crate::llvm::is_ci_llvm_modified(&parse("")) { - eprintln!("Detected LLVM as non-available: running in CI and modified LLVM in this change"); - return; - } - - let parse_llvm = |s| parse(s).llvm_from_ci; - let if_available = parse_llvm("llvm.download-ci-llvm = \"if-available\""); - - assert!(parse_llvm("llvm.download-ci-llvm = true")); - assert!(!parse_llvm("llvm.download-ci-llvm = false")); - assert_eq!(parse_llvm(""), if_available); - assert_eq!(parse_llvm("rust.channel = \"dev\""), if_available); - assert!(!parse_llvm("rust.channel = \"stable\"")); - assert!(parse_llvm("build.build = \"x86_64-unknown-linux-gnu\"")); - assert!(parse_llvm( - "llvm.assertions = true \r\n build.build = \"x86_64-unknown-linux-gnu\" \r\n llvm.download-ci-llvm = \"if-available\"" - )); - assert!(!parse_llvm( - "llvm.assertions = true \r\n build.build = \"aarch64-apple-darwin\" \r\n llvm.download-ci-llvm = \"if-available\"" - )); -} - -// FIXME(onur-ozkan): extend scope of the test -// refs: -// - https://github.com/rust-lang/rust/issues/109120 -// - https://github.com/rust-lang/rust/pull/109162#issuecomment-1496782487 -#[test] -fn detect_src_and_out() { - fn test(cfg: Config, build_dir: Option<&str>) { - // This will bring absolute form of `src/bootstrap` path - let current_dir = std::env::current_dir().unwrap(); - - // get `src` by moving into project root path - let expected_src = current_dir.ancestors().nth(2).unwrap(); - assert_eq!(&cfg.src, expected_src); - - // Sanity check for `src` - let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); - let expected_src = manifest_dir.ancestors().nth(2).unwrap(); - assert_eq!(&cfg.src, expected_src); - - // test if build-dir was manually given in config.toml - if let Some(custom_build_dir) = build_dir { - assert_eq!(&cfg.out, Path::new(custom_build_dir)); - } - // test the native bootstrap way - else { - // This should bring output path of bootstrap in absolute form - let cargo_target_dir = env::var_os("CARGO_TARGET_DIR").expect( - "CARGO_TARGET_DIR must been provided for the test environment from bootstrap", - ); - - // Move to `build` from `build/bootstrap` - let expected_out = Path::new(&cargo_target_dir).parent().unwrap(); - assert_eq!(&cfg.out, expected_out); - - let args: Vec = env::args().collect(); - - // Another test for `out` as a sanity check - // - // This will bring something similar to: - // `{build-dir}/bootstrap/debug/deps/bootstrap-c7ee91d5661e2804` - // `{build-dir}` can be anywhere, not just in the rust project directory. - let dep = Path::new(args.first().unwrap()); - let expected_out = dep.ancestors().nth(4).unwrap(); - - assert_eq!(&cfg.out, expected_out); - } - } - - test(parse(""), None); - - { - let build_dir = if cfg!(windows) { Some("C:\\tmp") } else { Some("/tmp") }; - test(parse("build.build-dir = \"/tmp\""), build_dir); - } -} - -#[test] -fn clap_verify() { - Flags::command().debug_assert(); -} - -#[test] -fn override_toml() { - let config = Config::parse_inner( - &[ - "check".to_owned(), - "--config=/does/not/exist".to_owned(), - "--set=changelog-seen=1".to_owned(), - "--set=rust.lto=fat".to_owned(), - "--set=rust.deny-warnings=false".to_owned(), - "--set=build.gdb=\"bar\"".to_owned(), - "--set=build.tools=[\"cargo\"]".to_owned(), - "--set=llvm.build-config={\"foo\" = \"bar\"}".to_owned(), - ], - |&_| { - toml::from_str( - r#" -changelog-seen = 0 -[rust] -lto = "off" -deny-warnings = true - -[build] -gdb = "foo" -tools = [] - -[llvm] -download-ci-llvm = false -build-config = {} - "#, - ) - .unwrap() - }, - ); - assert_eq!(config.changelog_seen, Some(1), "setting top-level value"); - assert_eq!( - config.rust_lto, - crate::config::RustcLto::Fat, - "setting string value without quotes" - ); - assert_eq!(config.gdb, Some("bar".into()), "setting string value with quotes"); - assert!(!config.deny_warnings, "setting boolean value"); - assert_eq!( - config.tools, - Some(["cargo".to_string()].into_iter().collect()), - "setting list value" - ); - assert_eq!( - config.llvm_build_config, - [("foo".to_string(), "bar".to_string())].into_iter().collect(), - "setting dictionary value" - ); -} - -#[test] -#[should_panic] -fn override_toml_duplicate() { - Config::parse_inner( - &[ - "check".to_owned(), - "--config=/does/not/exist".to_owned(), - "--set=changelog-seen=1".to_owned(), - "--set=changelog-seen=2".to_owned(), - ], - |&_| toml::from_str("changelog-seen = 0").unwrap(), - ); -} - -#[test] -fn profile_user_dist() { - fn get_toml(file: &Path) -> TomlConfig { - let contents = if file.ends_with("config.toml") { - "profile = \"user\"".to_owned() - } else { - assert!(file.ends_with("config.dist.toml")); - std::fs::read_to_string(dbg!(file)).unwrap() - }; - toml::from_str(&contents) - .and_then(|table: toml::Value| TomlConfig::deserialize(table)) - .unwrap() - } - Config::parse_inner(&["check".to_owned()], get_toml); -} - -#[test] -fn rust_optimize() { - assert!(parse("").rust_optimize.is_release()); - assert!(!parse("rust.optimize = false").rust_optimize.is_release()); - assert!(parse("rust.optimize = true").rust_optimize.is_release()); - assert!(!parse("rust.optimize = 0").rust_optimize.is_release()); - assert!(parse("rust.optimize = 1").rust_optimize.is_release()); - assert!(parse("rust.optimize = \"s\"").rust_optimize.is_release()); - assert_eq!(parse("rust.optimize = 1").rust_optimize.get_opt_level(), Some("1".to_string())); - assert_eq!(parse("rust.optimize = \"s\"").rust_optimize.get_opt_level(), Some("s".to_string())); -} - -#[test] -#[should_panic] -fn invalid_rust_optimize() { - parse("rust.optimize = \"a\""); -} diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py index f469dbea6..544a42d9a 100755 --- a/src/bootstrap/configure.py +++ b/src/bootstrap/configure.py @@ -59,6 +59,7 @@ o("missing-tools", "dist.missing-tools", "allow failures when building tools") o("use-libcxx", "llvm.use-libcxx", "build LLVM with libc++") o("control-flow-guard", "rust.control-flow-guard", "Enable Control Flow Guard") o("patch-binaries-for-nix", "build.patch-binaries-for-nix", "whether patch binaries for usage with Nix toolchains") +o("new-symbol-mangling", "rust.new-symbol-mangling", "use symbol-mangling-version v0") v("llvm-cflags", "llvm.cflags", "build LLVM with these extra compiler flags") v("llvm-cxxflags", "llvm.cxxflags", "build LLVM with these extra compiler flags") @@ -97,20 +98,7 @@ v("llvm-root", None, "set LLVM root") v("llvm-config", None, "set path to llvm-config") v("llvm-filecheck", None, "set path to LLVM's FileCheck utility") v("python", "build.python", "set path to python") -v("android-cross-path", "target.arm-linux-androideabi.android-ndk", - "Android NDK standalone path (deprecated)") -v("i686-linux-android-ndk", "target.i686-linux-android.android-ndk", - "i686-linux-android NDK standalone path") -v("arm-linux-androideabi-ndk", "target.arm-linux-androideabi.android-ndk", - "arm-linux-androideabi NDK standalone path") -v("armv7-linux-androideabi-ndk", "target.armv7-linux-androideabi.android-ndk", - "armv7-linux-androideabi NDK standalone path") -v("thumbv7neon-linux-androideabi-ndk", "target.thumbv7neon-linux-androideabi.android-ndk", - "thumbv7neon-linux-androideabi NDK standalone path") -v("aarch64-linux-android-ndk", "target.aarch64-linux-android.android-ndk", - "aarch64-linux-android NDK standalone path") -v("x86_64-linux-android-ndk", "target.x86_64-linux-android.android-ndk", - "x86_64-linux-android NDK standalone path") +v("android-ndk", "build.android-ndk", "set path to Android NDK") v("musl-root", "target.x86_64-unknown-linux-musl.musl-root", "MUSL root installation directory (deprecated)") v("musl-root-x86_64", "target.x86_64-unknown-linux-musl.musl-root", @@ -265,7 +253,7 @@ def parse_args(args): if not found: unknown_args.append(arg) - # Note: here and a few other places, we use [-1] to apply the *last* value + # NOTE: here and a few other places, we use [-1] to apply the *last* value # passed. But if option-checking is enabled, then the known_args loop will # also assert that options are only passed once. option_checking = ('option-checking' not in known_args @@ -489,7 +477,7 @@ def configure_section(lines, config): # These are used by rpm, but aren't accepted by x.py. # Give a warning that they're ignored, but not a hard error. if key in ["infodir", "localstatedir"]: - print("warning: {} will be ignored".format(key)) + print("WARNING: {} will be ignored".format(key)) else: raise RuntimeError("failed to find config line for {}".format(key)) diff --git a/src/bootstrap/defaults/config.codegen.toml b/src/bootstrap/defaults/config.codegen.toml index 113df88d7..7c33ce958 100644 --- a/src/bootstrap/defaults/config.codegen.toml +++ b/src/bootstrap/defaults/config.codegen.toml @@ -10,7 +10,7 @@ assertions = true # enable warnings during the llvm compilation enable-warnings = true # build llvm from source -download-ci-llvm = false +download-ci-llvm = "if-unchanged" [rust] # This enables `RUSTC_LOG=debug`, avoiding confusing situations diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs deleted file mode 100644 index 32da4ac29..000000000 --- a/src/bootstrap/dist.rs +++ /dev/null @@ -1,2274 +0,0 @@ -//! Implementation of the various distribution aspects of the compiler. -//! -//! This module is responsible for creating tarballs of the standard library, -//! compiler, and documentation. This ends up being what we distribute to -//! everyone as well. -//! -//! No tarball is actually created literally in this file, but rather we shell -//! out to `rust-installer` still. This may one day be replaced with bits and -//! pieces of `rustup.rs`! - -use std::collections::HashSet; -use std::env; -use std::ffi::OsStr; -use std::fs; -use std::io::Write; -use std::path::{Path, PathBuf}; -use std::process::Command; - -use object::read::archive::ArchiveFile; -use object::BinaryFormat; - -use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; -use crate::cache::{Interned, INTERNER}; -use crate::channel; -use crate::compile; -use crate::config::TargetSelection; -use crate::doc::DocumentationFormat; -use crate::llvm; -use crate::tarball::{GeneratedTarball, OverlayKind, Tarball}; -use crate::tool::{self, Tool}; -use crate::util::{exe, is_dylib, output, t, timeit}; -use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS}; - -pub fn pkgname(builder: &Builder<'_>, component: &str) -> String { - format!("{}-{}", component, builder.rust_package_vers()) -} - -pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf { - builder.out.join("dist") -} - -pub fn tmpdir(builder: &Builder<'_>) -> PathBuf { - builder.out.join("tmp/dist") -} - -fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool { - if !builder.config.extended { - return false; - } - builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool)) -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Docs { - pub host: TargetSelection, -} - -impl Step for Docs { - type Output = Option; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = run.builder.config.docs; - run.alias("rust-docs").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Docs { host: run.target }); - } - - /// Builds the `rust-docs` installer component. - fn run(self, builder: &Builder<'_>) -> Option { - let host = self.host; - builder.default_doc(&[]); - - let dest = "share/doc/rust/html"; - - let mut tarball = Tarball::new(builder, "rust-docs", &host.triple); - tarball.set_product_name("Rust Documentation"); - tarball.add_bulk_dir(&builder.doc_out(host), dest); - tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644); - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct JsonDocs { - pub host: TargetSelection, -} - -impl Step for JsonDocs { - type Output = Option; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = run.builder.config.docs; - run.alias("rust-docs-json").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(JsonDocs { host: run.target }); - } - - /// Builds the `rust-docs-json` installer component. - fn run(self, builder: &Builder<'_>) -> Option { - let host = self.host; - builder.ensure(crate::doc::Std::new( - builder.top_stage, - host, - builder, - DocumentationFormat::JSON, - )); - - let dest = "share/doc/rust/json"; - - let mut tarball = Tarball::new(builder, "rust-docs-json", &host.triple); - tarball.set_product_name("Rust Documentation In JSON Format"); - tarball.is_preview(true); - tarball.add_bulk_dir(&builder.json_doc_out(host), dest); - Some(tarball.generate()) - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct RustcDocs { - pub host: TargetSelection, -} - -impl Step for RustcDocs { - type Output = Option; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.alias("rustc-docs").default_condition(builder.config.compiler_docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustcDocs { host: run.target }); - } - - /// Builds the `rustc-docs` installer component. - fn run(self, builder: &Builder<'_>) -> Option { - let host = self.host; - builder.default_doc(&[]); - - let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple); - tarball.set_product_name("Rustc Documentation"); - tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc"); - Some(tarball.generate()) - } -} - -fn find_files(files: &[&str], path: &[PathBuf]) -> Vec { - let mut found = Vec::with_capacity(files.len()); - - for file in files { - let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists()); - - if let Some(file_path) = file_path { - found.push(file_path); - } else { - panic!("Could not find '{file}' in {path:?}"); - } - } - - found -} - -fn make_win_dist( - rust_root: &Path, - plat_root: &Path, - target: TargetSelection, - builder: &Builder<'_>, -) { - if builder.config.dry_run() { - return; - } - - //Ask gcc where it keeps its stuff - let mut cmd = Command::new(builder.cc(target)); - cmd.arg("-print-search-dirs"); - let gcc_out = output(&mut cmd); - - let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect(); - let mut lib_path = Vec::new(); - - for line in gcc_out.lines() { - let idx = line.find(':').unwrap(); - let key = &line[..idx]; - let trim_chars: &[_] = &[' ', '=']; - let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars)); - - if key == "programs" { - bin_path.extend(value); - } else if key == "libraries" { - lib_path.extend(value); - } - } - - let compiler = if target == "i686-pc-windows-gnu" { - "i686-w64-mingw32-gcc.exe" - } else if target == "x86_64-pc-windows-gnu" { - "x86_64-w64-mingw32-gcc.exe" - } else { - "gcc.exe" - }; - let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"]; - let mut rustc_dlls = vec!["libwinpthread-1.dll"]; - if target.starts_with("i686-") { - rustc_dlls.push("libgcc_s_dw2-1.dll"); - } else { - rustc_dlls.push("libgcc_s_seh-1.dll"); - } - - // Libraries necessary to link the windows-gnu toolchains. - // System libraries will be preferred if they are available (see #67429). - let target_libs = [ - //MinGW libs - "libgcc.a", - "libgcc_eh.a", - "libgcc_s.a", - "libm.a", - "libmingw32.a", - "libmingwex.a", - "libstdc++.a", - "libiconv.a", - "libmoldname.a", - "libpthread.a", - //Windows import libs - //This should contain only the set of libraries necessary to link the standard library. - "libadvapi32.a", - "libbcrypt.a", - "libcomctl32.a", - "libcomdlg32.a", - "libcredui.a", - "libcrypt32.a", - "libdbghelp.a", - "libgdi32.a", - "libimagehlp.a", - "libiphlpapi.a", - "libkernel32.a", - "libmsimg32.a", - "libmsvcrt.a", - "libntdll.a", - "libodbc32.a", - "libole32.a", - "liboleaut32.a", - "libopengl32.a", - "libpsapi.a", - "librpcrt4.a", - "libsecur32.a", - "libsetupapi.a", - "libshell32.a", - "libsynchronization.a", - "libuser32.a", - "libuserenv.a", - "libuuid.a", - "libwinhttp.a", - "libwinmm.a", - "libwinspool.a", - "libws2_32.a", - "libwsock32.a", - ]; - - //Find mingw artifacts we want to bundle - let target_tools = find_files(&target_tools, &bin_path); - let rustc_dlls = find_files(&rustc_dlls, &bin_path); - let target_libs = find_files(&target_libs, &lib_path); - - // Copy runtime dlls next to rustc.exe - let dist_bin_dir = rust_root.join("bin/"); - fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed"); - for src in rustc_dlls { - builder.copy_to_folder(&src, &dist_bin_dir); - } - - //Copy platform tools to platform-specific bin directory - let target_bin_dir = plat_root - .join("lib") - .join("rustlib") - .join(target.triple) - .join("bin") - .join("self-contained"); - fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed"); - for src in target_tools { - builder.copy_to_folder(&src, &target_bin_dir); - } - - // Warn windows-gnu users that the bundled GCC cannot compile C files - builder.create( - &target_bin_dir.join("GCC-WARNING.txt"), - "gcc.exe contained in this folder cannot be used for compiling C files - it is only \ - used as a linker. In order to be able to compile projects containing C code use \ - the GCC provided by MinGW or Cygwin.", - ); - - //Copy platform libs to platform-specific lib directory - let target_lib_dir = plat_root - .join("lib") - .join("rustlib") - .join(target.triple) - .join("lib") - .join("self-contained"); - fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed"); - for src in target_libs { - builder.copy_to_folder(&src, &target_lib_dir); - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Mingw { - pub host: TargetSelection, -} - -impl Step for Mingw { - type Output = Option; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("rust-mingw") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Mingw { host: run.target }); - } - - /// Builds the `rust-mingw` installer component. - /// - /// This contains all the bits and pieces to run the MinGW Windows targets - /// without any extra installed software (e.g., we bundle gcc, libraries, etc). - fn run(self, builder: &Builder<'_>) -> Option { - let host = self.host; - if !host.ends_with("pc-windows-gnu") || !builder.config.dist_include_mingw_linker { - return None; - } - - let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple); - tarball.set_product_name("Rust MinGW"); - - // The first argument is a "temporary directory" which is just - // thrown away (this contains the runtime DLLs included in the rustc package - // above) and the second argument is where to place all the MinGW components - // (which is what we want). - make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder); - - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Rustc { - pub compiler: Compiler, -} - -impl Step for Rustc { - type Output = GeneratedTarball; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("rustc") - } - - fn make_run(run: RunConfig<'_>) { - run.builder - .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) }); - } - - /// Creates the `rustc` installer component. - fn run(self, builder: &Builder<'_>) -> GeneratedTarball { - let compiler = self.compiler; - let host = self.compiler.host; - - let tarball = Tarball::new(builder, "rustc", &host.triple); - - // Prepare the rustc "image", what will actually end up getting installed - prepare_image(builder, compiler, tarball.image_dir()); - - // On MinGW we've got a few runtime DLL dependencies that we need to - // include. The first argument to this script is where to put these DLLs - // (the image we're creating), and the second argument is a junk directory - // to ignore all other MinGW stuff the script creates. - // - // On 32-bit MinGW we're always including a DLL which needs some extra - // licenses to distribute. On 64-bit MinGW we don't actually distribute - // anything requiring us to distribute a license, but it's likely the - // install will *also* include the rust-mingw package, which also needs - // licenses, so to be safe we just include it here in all MinGW packages. - if host.ends_with("pc-windows-gnu") && builder.config.dist_include_mingw_linker { - make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder); - tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc"); - } - - return tarball.generate(); - - fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) { - let host = compiler.host; - let src = builder.sysroot(compiler); - - // Copy rustc/rustdoc binaries - t!(fs::create_dir_all(image.join("bin"))); - builder.cp_r(&src.join("bin"), &image.join("bin")); - - if builder - .config - .tools - .as_ref() - .map_or(true, |tools| tools.iter().any(|tool| tool == "rustdoc")) - { - let rustdoc = builder.rustdoc(compiler); - builder.install(&rustdoc, &image.join("bin"), 0o755); - } - - if let Some(ra_proc_macro_srv) = builder.ensure_if_default( - tool::RustAnalyzerProcMacroSrv { - compiler: builder.compiler_for( - compiler.stage, - builder.config.build, - compiler.host, - ), - target: compiler.host, - }, - builder.kind, - ) { - builder.install(&ra_proc_macro_srv, &image.join("libexec"), 0o755); - } - - let libdir_relative = builder.libdir_relative(compiler); - - // Copy runtime DLLs needed by the compiler - if libdir_relative.to_str() != Some("bin") { - let libdir = builder.rustc_libdir(compiler); - for entry in builder.read_dir(&libdir) { - let name = entry.file_name(); - if let Some(s) = name.to_str() { - if is_dylib(s) { - // Don't use custom libdir here because ^lib/ will be resolved again - // with installer - builder.install(&entry.path(), &image.join("lib"), 0o644); - } - } - } - } - - // Copy over the codegen backends - let backends_src = builder.sysroot_codegen_backends(compiler); - let backends_rel = backends_src - .strip_prefix(&src) - .unwrap() - .strip_prefix(builder.sysroot_libdir_relative(compiler)) - .unwrap(); - // Don't use custom libdir here because ^lib/ will be resolved again with installer - let backends_dst = image.join("lib").join(&backends_rel); - - t!(fs::create_dir_all(&backends_dst)); - builder.cp_r(&backends_src, &backends_dst); - - // Copy libLLVM.so to the lib dir as well, if needed. While not - // technically needed by rustc itself it's needed by lots of other - // components like the llvm tools and LLD. LLD is included below and - // tools/LLDB come later, so let's just throw it in the rustc - // component for now. - maybe_install_llvm_runtime(builder, host, image); - - let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin"); - t!(fs::create_dir_all(&dst_dir)); - - // Copy over lld if it's there - if builder.config.lld_enabled { - let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin"); - let rust_lld = exe("rust-lld", compiler.host); - builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld)); - // for `-Z gcc-ld=lld` - let gcc_lld_src_dir = src_dir.join("gcc-ld"); - let gcc_lld_dst_dir = dst_dir.join("gcc-ld"); - t!(fs::create_dir(&gcc_lld_dst_dir)); - for name in crate::LLD_FILE_NAMES { - let exe_name = exe(name, compiler.host); - builder - .copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name)); - } - } - - // Man pages - t!(fs::create_dir_all(image.join("share/man/man1"))); - let man_src = builder.src.join("src/doc/man"); - let man_dst = image.join("share/man/man1"); - - // don't use our `bootstrap::util::{copy, cp_r}`, because those try - // to hardlink, and we don't want to edit the source templates - for file_entry in builder.read_dir(&man_src) { - let page_src = file_entry.path(); - let page_dst = man_dst.join(file_entry.file_name()); - let src_text = t!(std::fs::read_to_string(&page_src)); - let new_text = src_text.replace("", &builder.version); - t!(std::fs::write(&page_dst, &new_text)); - t!(fs::copy(&page_src, &page_dst)); - } - - // Debugger scripts - builder - .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host }); - - // Misc license info - let cp = |file: &str| { - builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644); - }; - cp("COPYRIGHT"); - cp("LICENSE-APACHE"); - cp("LICENSE-MIT"); - cp("README.md"); - } - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct DebuggerScripts { - pub sysroot: Interned, - pub host: TargetSelection, -} - -impl Step for DebuggerScripts { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Copies debugger scripts for `target` into the `sysroot` specified. - fn run(self, builder: &Builder<'_>) { - let host = self.host; - let sysroot = self.sysroot; - let dst = sysroot.join("lib/rustlib/etc"); - t!(fs::create_dir_all(&dst)); - let cp_debugger_script = |file: &str| { - builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644); - }; - if host.contains("windows-msvc") { - // windbg debugger scripts - builder.install( - &builder.src.join("src/etc/rust-windbg.cmd"), - &sysroot.join("bin"), - 0o755, - ); - - cp_debugger_script("natvis/intrinsic.natvis"); - cp_debugger_script("natvis/liballoc.natvis"); - cp_debugger_script("natvis/libcore.natvis"); - cp_debugger_script("natvis/libstd.natvis"); - } else { - cp_debugger_script("rust_types.py"); - - // gdb debugger scripts - builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755); - builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755); - - cp_debugger_script("gdb_load_rust_pretty_printers.py"); - cp_debugger_script("gdb_lookup.py"); - cp_debugger_script("gdb_providers.py"); - - // lldb debugger scripts - builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755); - - cp_debugger_script("lldb_lookup.py"); - cp_debugger_script("lldb_providers.py"); - cp_debugger_script("lldb_commands") - } - } -} - -fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool { - // The only true set of target libraries came from the build triple, so - // let's reduce redundant work by only producing archives from that host. - if compiler.host != builder.config.build { - builder.info("\tskipping, not a build host"); - true - } else { - false - } -} - -/// Check that all objects in rlibs for UEFI targets are COFF. This -/// ensures that the C compiler isn't producing ELF objects, which would -/// not link correctly with the COFF objects. -fn verify_uefi_rlib_format(builder: &Builder<'_>, target: TargetSelection, stamp: &Path) { - if !target.ends_with("-uefi") { - return; - } - - for (path, _) in builder.read_stamp_file(stamp) { - if path.extension() != Some(OsStr::new("rlib")) { - continue; - } - - let data = t!(fs::read(&path)); - let data = data.as_slice(); - let archive = t!(ArchiveFile::parse(data)); - for member in archive.members() { - let member = t!(member); - let member_data = t!(member.data(data)); - - let is_coff = match object::File::parse(member_data) { - Ok(member_file) => member_file.format() == BinaryFormat::Coff, - Err(_) => false, - }; - - if !is_coff { - let member_name = String::from_utf8_lossy(member.name()); - panic!("member {} in {} is not COFF", member_name, path.display()); - } - } - } -} - -/// Copy stamped files into an image's `target/lib` directory. -fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) { - let dst = image.join("lib/rustlib").join(target.triple).join("lib"); - let self_contained_dst = dst.join("self-contained"); - t!(fs::create_dir_all(&dst)); - t!(fs::create_dir_all(&self_contained_dst)); - for (path, dependency_type) in builder.read_stamp_file(stamp) { - if dependency_type == DependencyType::TargetSelfContained { - builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap())); - } else if dependency_type == DependencyType::Target || builder.config.build == target { - builder.copy(&path, &dst.join(path.file_name().unwrap())); - } - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Std { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Std { - type Output = Option; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("rust-std") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Std { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - - if skip_host_target_lib(builder, compiler) { - return None; - } - - builder.ensure(compile::Std::new(compiler, target)); - - let mut tarball = Tarball::new(builder, "rust-std", &target.triple); - tarball.include_target_in_component_name(true); - - let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); - let stamp = compile::libstd_stamp(builder, compiler_to_use, target); - verify_uefi_rlib_format(builder, target, &stamp); - copy_target_libs(builder, target, &tarball.image_dir(), &stamp); - - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct RustcDev { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for RustcDev { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("rustc-dev") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustcDev { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - if skip_host_target_lib(builder, compiler) { - return None; - } - - builder.ensure(compile::Rustc::new(compiler, target)); - - let tarball = Tarball::new(builder, "rustc-dev", &target.triple); - - let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); - let stamp = compile::librustc_stamp(builder, compiler_to_use, target); - copy_target_libs(builder, target, tarball.image_dir(), &stamp); - - let src_files = &["Cargo.lock"]; - // This is the reduced set of paths which will become the rustc-dev component - // (essentially the compiler crates and all of their path dependencies). - copy_src_dirs( - builder, - &builder.src, - &["compiler"], - &[], - &tarball.image_dir().join("lib/rustlib/rustc-src/rust"), - ); - for file in src_files { - tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644); - } - - Some(tarball.generate()) - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Analysis { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Analysis { - type Output = Option; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(&run.builder, "analysis"); - run.alias("rust-analysis").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Analysis { - // Find the actual compiler (handling the full bootstrap option) which - // produced the save-analysis data because that data isn't copied - // through the sysroot uplifting. - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - /// Creates a tarball of (degenerate) save-analysis metadata, if available. - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - if compiler.host != builder.config.build { - return None; - } - - let src = builder - .stage_out(compiler, Mode::Std) - .join(target.triple) - .join(builder.cargo_dir()) - .join("deps") - .join("save-analysis"); - - // Write a file indicating that this component has been removed. - t!(std::fs::create_dir_all(&src)); - let mut removed = src.clone(); - removed.push("removed.json"); - let mut f = t!(std::fs::File::create(removed)); - t!(write!(f, r#"{{ "warning": "The `rust-analysis` component has been removed." }}"#)); - - let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple); - tarball.include_target_in_component_name(true); - tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple)); - Some(tarball.generate()) - } -} - -/// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to -/// `dst_dir`. -fn copy_src_dirs( - builder: &Builder<'_>, - base: &Path, - src_dirs: &[&str], - exclude_dirs: &[&str], - dst_dir: &Path, -) { - fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool { - let spath = match path.to_str() { - Some(path) => path, - None => return false, - }; - if spath.ends_with('~') || spath.ends_with(".pyc") { - return false; - } - - const LLVM_PROJECTS: &[&str] = &[ - "llvm-project/clang", - "llvm-project\\clang", - "llvm-project/libunwind", - "llvm-project\\libunwind", - "llvm-project/lld", - "llvm-project\\lld", - "llvm-project/lldb", - "llvm-project\\lldb", - "llvm-project/llvm", - "llvm-project\\llvm", - "llvm-project/compiler-rt", - "llvm-project\\compiler-rt", - "llvm-project/cmake", - "llvm-project\\cmake", - "llvm-project/runtimes", - "llvm-project\\runtimes", - ]; - if spath.contains("llvm-project") - && !spath.ends_with("llvm-project") - && !LLVM_PROJECTS.iter().any(|path| spath.contains(path)) - { - return false; - } - - const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"]; - if LLVM_TEST.iter().any(|path| spath.contains(path)) - && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s")) - { - return false; - } - - let full_path = Path::new(dir).join(path); - if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) { - return false; - } - - let excludes = [ - "CVS", - "RCS", - "SCCS", - ".git", - ".gitignore", - ".gitmodules", - ".gitattributes", - ".cvsignore", - ".svn", - ".arch-ids", - "{arch}", - "=RELEASE-ID", - "=meta-update", - "=update", - ".bzr", - ".bzrignore", - ".bzrtags", - ".hg", - ".hgignore", - ".hgrags", - "_darcs", - ]; - !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s)) - } - - // Copy the directories using our filter - for item in src_dirs { - let dst = &dst_dir.join(item); - t!(fs::create_dir_all(dst)); - builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path)); - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Src; - -impl Step for Src { - /// The output path of the src installer tarball - type Output = GeneratedTarball; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("rust-src") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Src); - } - - /// Creates the `rust-src` installer component - fn run(self, builder: &Builder<'_>) -> GeneratedTarball { - if !builder.config.dry_run() { - builder.update_submodule(&Path::new("src/llvm-project")); - } - - let tarball = Tarball::new_targetless(builder, "rust-src"); - - // A lot of tools expect the rust-src component to be entirely in this directory, so if you - // change that (e.g. by adding another directory `lib/rustlib/src/foo` or - // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions - // and fix them... - // - // NOTE: if you update the paths here, you also should update the "virtual" path - // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs` - let dst_src = tarball.image_dir().join("lib/rustlib/src/rust"); - - let src_files = ["Cargo.lock"]; - // This is the reduced set of paths which will become the rust-src component - // (essentially libstd and all of its path dependencies). - copy_src_dirs( - builder, - &builder.src, - &["library", "src/llvm-project/libunwind"], - &[ - // not needed and contains symlinks which rustup currently - // chokes on when unpacking. - "library/backtrace/crates", - // these are 30MB combined and aren't necessary for building - // the standard library. - "library/stdarch/Cargo.toml", - "library/stdarch/crates/stdarch-verify", - "library/stdarch/crates/intrinsic-test", - ], - &dst_src, - ); - for file in src_files.iter() { - builder.copy(&builder.src.join(file), &dst_src.join(file)); - } - - tarball.generate() - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct PlainSourceTarball; - -impl Step for PlainSourceTarball { - /// Produces the location of the tarball generated - type Output = GeneratedTarball; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.alias("rustc-src").default_condition(builder.config.rust_dist_src) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(PlainSourceTarball); - } - - /// Creates the plain source tarball - fn run(self, builder: &Builder<'_>) -> GeneratedTarball { - // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which - // means neither rustup nor rustup-toolchain-install-master know how to download it. - // It also contains symbolic links, unlike other any other dist tarball. - // It's used for distros building rustc from source in a pre-vendored environment. - let mut tarball = Tarball::new(builder, "rustc", "src"); - tarball.permit_symlinks(true); - let plain_dst_src = tarball.image_dir(); - - // This is the set of root paths which will become part of the source package - let src_files = [ - "COPYRIGHT", - "LICENSE-APACHE", - "LICENSE-MIT", - "CONTRIBUTING.md", - "README.md", - "RELEASES.md", - "configure", - "x.py", - "config.example.toml", - "Cargo.toml", - "Cargo.lock", - ".gitmodules", - ]; - let src_dirs = ["src", "compiler", "library", "tests"]; - - copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src); - - // Copy the files normally - for item in &src_files { - builder.copy(&builder.src.join(item), &plain_dst_src.join(item)); - } - - // Create the version file - builder.create(&plain_dst_src.join("version"), &builder.rust_version()); - if let Some(info) = builder.rust_info().info() { - channel::write_commit_hash_file(&plain_dst_src, &info.sha); - channel::write_commit_info_file(&plain_dst_src, info); - } - - // If we're building from git sources, we need to vendor a complete distribution. - if builder.rust_info().is_managed_git_subrepository() { - // Ensure we have the submodules checked out. - builder.update_submodule(Path::new("src/tools/cargo")); - builder.update_submodule(Path::new("src/tools/rust-analyzer")); - - // Vendor all Cargo dependencies - let mut cmd = Command::new(&builder.initial_cargo); - cmd.arg("vendor") - .arg("--sync") - .arg(builder.src.join("./src/tools/cargo/Cargo.toml")) - .arg("--sync") - .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml")) - .arg("--sync") - .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml")) - .arg("--sync") - .arg(builder.src.join("./src/bootstrap/Cargo.toml")) - // Will read the libstd Cargo.toml - // which uses the unstable `public-dependency` feature. - .env("RUSTC_BOOTSTRAP", "1") - .current_dir(&plain_dst_src); - - let config = if !builder.config.dry_run() { - t!(String::from_utf8(t!(cmd.output()).stdout)) - } else { - String::new() - }; - - let cargo_config_dir = plain_dst_src.join(".cargo"); - builder.create_dir(&cargo_config_dir); - builder.create(&cargo_config_dir.join("config.toml"), &config); - } - - tarball.bare() - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Cargo { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Cargo { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(&run.builder, "cargo"); - run.alias("cargo").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Cargo { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - - let cargo = builder.ensure(tool::Cargo { compiler, target }); - let src = builder.src.join("src/tools/cargo"); - let etc = src.join("src/etc"); - - // Prepare the image directory - let mut tarball = Tarball::new(builder, "cargo", &target.triple); - tarball.set_overlay(OverlayKind::Cargo); - - tarball.add_file(&cargo, "bin", 0o755); - tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644); - tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo"); - tarball.add_dir(etc.join("man"), "share/man/man1"); - tarball.add_legal_and_readme_to("share/doc/cargo"); - - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Rls { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Rls { - type Output = Option; - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(&run.builder, "rls"); - run.alias("rls").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Rls { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - - let rls = builder - .ensure(tool::Rls { compiler, target, extra_features: Vec::new() }) - .expect("rls expected to build"); - - let mut tarball = Tarball::new(builder, "rls", &target.triple); - tarball.set_overlay(OverlayKind::RLS); - tarball.is_preview(true); - tarball.add_file(rls, "bin", 0o755); - tarball.add_legal_and_readme_to("share/doc/rls"); - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct RustAnalyzer { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for RustAnalyzer { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(&run.builder, "rust-analyzer"); - run.alias("rust-analyzer").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustAnalyzer { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - - let rust_analyzer = builder - .ensure(tool::RustAnalyzer { compiler, target }) - .expect("rust-analyzer always builds"); - - let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple); - tarball.set_overlay(OverlayKind::RustAnalyzer); - tarball.is_preview(true); - tarball.add_file(rust_analyzer, "bin", 0o755); - tarball.add_legal_and_readme_to("share/doc/rust-analyzer"); - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Clippy { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Clippy { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(&run.builder, "clippy"); - run.alias("clippy").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Clippy { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - - // Prepare the image directory - // We expect clippy to build, because we've exited this step above if tool - // state for clippy isn't testing. - let clippy = builder - .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() }) - .expect("clippy expected to build - essential tool"); - let cargoclippy = builder - .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() }) - .expect("clippy expected to build - essential tool"); - - let mut tarball = Tarball::new(builder, "clippy", &target.triple); - tarball.set_overlay(OverlayKind::Clippy); - tarball.is_preview(true); - tarball.add_file(clippy, "bin", 0o755); - tarball.add_file(cargoclippy, "bin", 0o755); - tarball.add_legal_and_readme_to("share/doc/clippy"); - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Miri { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Miri { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(&run.builder, "miri"); - run.alias("miri").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Miri { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - // This prevents miri from being built for "dist" or "install" - // on the stable/beta channels. It is a nightly-only tool and should - // not be included. - if !builder.build.unstable_features() { - return None; - } - let compiler = self.compiler; - let target = self.target; - - let miri = builder.ensure(tool::Miri { compiler, target, extra_features: Vec::new() })?; - let cargomiri = - builder.ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })?; - - let mut tarball = Tarball::new(builder, "miri", &target.triple); - tarball.set_overlay(OverlayKind::Miri); - tarball.is_preview(true); - tarball.add_file(miri, "bin", 0o755); - tarball.add_file(cargomiri, "bin", 0o755); - tarball.add_legal_and_readme_to("share/doc/miri"); - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Rustfmt { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Rustfmt { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(&run.builder, "rustfmt"); - run.alias("rustfmt").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Rustfmt { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - - let rustfmt = builder - .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() }) - .expect("rustfmt expected to build - essential tool"); - let cargofmt = builder - .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() }) - .expect("cargo fmt expected to build - essential tool"); - let mut tarball = Tarball::new(builder, "rustfmt", &target.triple); - tarball.set_overlay(OverlayKind::Rustfmt); - tarball.is_preview(true); - tarball.add_file(rustfmt, "bin", 0o755); - tarball.add_file(cargofmt, "bin", 0o755); - tarball.add_legal_and_readme_to("share/doc/rustfmt"); - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct RustDemangler { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for RustDemangler { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - // While other tools use `should_build_extended_tool` to decide whether to be run by - // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like - // the other ones or if `profiler = true`. Because we don't know the target at this stage - // we run the step by default when only `extended = true`, and decide whether to actually - // run it or not later. - let default = run.builder.config.extended; - run.alias("rust-demangler").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustDemangler { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - - // Only build this extended tool if explicitly included in `tools`, or if `profiler = true` - let condition = should_build_extended_tool(builder, "rust-demangler") - || builder.config.profiler_enabled(target); - if builder.config.extended && !condition { - return None; - } - - let rust_demangler = builder - .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() }) - .expect("rust-demangler expected to build - in-tree tool"); - - // Prepare the image directory - let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple); - tarball.set_overlay(OverlayKind::RustDemangler); - tarball.is_preview(true); - tarball.add_file(&rust_demangler, "bin", 0o755); - tarball.add_legal_and_readme_to("share/doc/rust-demangler"); - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Extended { - stage: u32, - host: TargetSelection, - target: TargetSelection, -} - -impl Step for Extended { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.alias("extended").default_condition(builder.config.extended) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Extended { - stage: run.builder.top_stage, - host: run.builder.config.build, - target: run.target, - }); - } - - /// Creates a combined installer for the specified target in the provided stage. - fn run(self, builder: &Builder<'_>) { - let target = self.target; - let stage = self.stage; - let compiler = builder.compiler_for(self.stage, self.host, self.target); - - builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target)); - - let mut tarballs = Vec::new(); - let mut built_tools = HashSet::new(); - macro_rules! add_component { - ($name:expr => $step:expr) => { - if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) { - tarballs.push(tarball); - built_tools.insert($name); - } - }; - } - - // When rust-std package split from rustc, we needed to ensure that during - // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering - // the std files during uninstall. To do this ensure that rustc comes - // before rust-std in the list below. - tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) })); - tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std")); - - if target.ends_with("windows-gnu") { - tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw")); - } - - add_component!("rust-docs" => Docs { host: target }); - add_component!("rust-json-docs" => JsonDocs { host: target }); - add_component!("rust-demangler"=> RustDemangler { compiler, target }); - add_component!("cargo" => Cargo { compiler, target }); - add_component!("rustfmt" => Rustfmt { compiler, target }); - add_component!("rls" => Rls { compiler, target }); - add_component!("rust-analyzer" => RustAnalyzer { compiler, target }); - add_component!("llvm-components" => LlvmTools { target }); - add_component!("clippy" => Clippy { compiler, target }); - add_component!("miri" => Miri { compiler, target }); - add_component!("analysis" => Analysis { compiler, target }); - - let etc = builder.src.join("src/etc/installer"); - - // Avoid producing tarballs during a dry run. - if builder.config.dry_run() { - return; - } - - let tarball = Tarball::new(builder, "rust", &target.triple); - let generated = tarball.combine(&tarballs); - - let tmp = tmpdir(builder).join("combined-tarball"); - let work = generated.work_dir(); - - let mut license = String::new(); - license += &builder.read(&builder.src.join("COPYRIGHT")); - license += &builder.read(&builder.src.join("LICENSE-APACHE")); - license += &builder.read(&builder.src.join("LICENSE-MIT")); - license.push('\n'); - license.push('\n'); - - let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18"; - let mut rtf = rtf.to_string(); - rtf.push('\n'); - for line in license.lines() { - rtf.push_str(line); - rtf.push_str("\\line "); - } - rtf.push('}'); - - fn filter(contents: &str, marker: &str) -> String { - let start = format!("tool-{marker}-start"); - let end = format!("tool-{marker}-end"); - let mut lines = Vec::new(); - let mut omitted = false; - for line in contents.lines() { - if line.contains(&start) { - omitted = true; - } else if line.contains(&end) { - omitted = false; - } else if !omitted { - lines.push(line); - } - } - - lines.join("\n") - } - - let xform = |p: &Path| { - let mut contents = t!(fs::read_to_string(p)); - for tool in &["rust-demangler", "miri", "rust-docs"] { - if !built_tools.contains(tool) { - contents = filter(&contents, tool); - } - } - let ret = tmp.join(p.file_name().unwrap()); - t!(fs::write(&ret, &contents)); - ret - }; - - if target.contains("apple-darwin") { - builder.info("building pkg installer"); - let pkg = tmp.join("pkg"); - let _ = fs::remove_dir_all(&pkg); - - let pkgbuild = |component: &str| { - let mut cmd = Command::new("pkgbuild"); - cmd.arg("--identifier") - .arg(format!("org.rust-lang.{}", component)) - .arg("--scripts") - .arg(pkg.join(component)) - .arg("--nopayload") - .arg(pkg.join(component).with_extension("pkg")); - builder.run(&mut cmd); - }; - - let prepare = |name: &str| { - builder.create_dir(&pkg.join(name)); - builder.cp_r( - &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)), - &pkg.join(name), - ); - builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755); - pkgbuild(name); - }; - prepare("rustc"); - prepare("cargo"); - prepare("rust-std"); - prepare("rust-analysis"); - prepare("clippy"); - prepare("rust-analyzer"); - for tool in &["rust-docs", "rust-demangler", "miri"] { - if built_tools.contains(tool) { - prepare(tool); - } - } - // create an 'uninstall' package - builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755); - pkgbuild("uninstall"); - - builder.create_dir(&pkg.join("res")); - builder.create(&pkg.join("res/LICENSE.txt"), &license); - builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644); - let mut cmd = Command::new("productbuild"); - cmd.arg("--distribution") - .arg(xform(&etc.join("pkg/Distribution.xml"))) - .arg("--resources") - .arg(pkg.join("res")) - .arg(distdir(builder).join(format!( - "{}-{}.pkg", - pkgname(builder, "rust"), - target.triple - ))) - .arg("--package-path") - .arg(&pkg); - let _time = timeit(builder); - builder.run(&mut cmd); - } - - if target.contains("windows") { - let exe = tmp.join("exe"); - let _ = fs::remove_dir_all(&exe); - - let prepare = |name: &str| { - builder.create_dir(&exe.join(name)); - let dir = if name == "rust-std" || name == "rust-analysis" { - format!("{}-{}", name, target.triple) - } else if name == "rust-analyzer" { - "rust-analyzer-preview".to_string() - } else if name == "clippy" { - "clippy-preview".to_string() - } else if name == "rust-demangler" { - "rust-demangler-preview".to_string() - } else if name == "miri" { - "miri-preview".to_string() - } else { - name.to_string() - }; - builder.cp_r( - &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir), - &exe.join(name), - ); - builder.remove(&exe.join(name).join("manifest.in")); - }; - prepare("rustc"); - prepare("cargo"); - prepare("rust-analysis"); - prepare("rust-std"); - for tool in &["clippy", "rust-analyzer", "rust-docs", "rust-demangler", "miri"] { - if built_tools.contains(tool) { - prepare(tool); - } - } - if target.ends_with("windows-gnu") { - prepare("rust-mingw"); - } - - builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644); - - // Generate msi installer - let wix_path = env::var_os("WIX") - .expect("`WIX` environment variable must be set for generating MSI installer(s)."); - let wix = PathBuf::from(wix_path); - let heat = wix.join("bin/heat.exe"); - let candle = wix.join("bin/candle.exe"); - let light = wix.join("bin/light.exe"); - - let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"]; - builder.run( - Command::new(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rustc") - .args(&heat_flags) - .arg("-cg") - .arg("RustcGroup") - .arg("-dr") - .arg("Rustc") - .arg("-var") - .arg("var.RustcDir") - .arg("-out") - .arg(exe.join("RustcGroup.wxs")), - ); - if built_tools.contains("rust-docs") { - builder.run( - Command::new(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rust-docs") - .args(&heat_flags) - .arg("-cg") - .arg("DocsGroup") - .arg("-dr") - .arg("Docs") - .arg("-var") - .arg("var.DocsDir") - .arg("-out") - .arg(exe.join("DocsGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/squash-components.xsl")), - ); - } - builder.run( - Command::new(&heat) - .current_dir(&exe) - .arg("dir") - .arg("cargo") - .args(&heat_flags) - .arg("-cg") - .arg("CargoGroup") - .arg("-dr") - .arg("Cargo") - .arg("-var") - .arg("var.CargoDir") - .arg("-out") - .arg(exe.join("CargoGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/remove-duplicates.xsl")), - ); - builder.run( - Command::new(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rust-std") - .args(&heat_flags) - .arg("-cg") - .arg("StdGroup") - .arg("-dr") - .arg("Std") - .arg("-var") - .arg("var.StdDir") - .arg("-out") - .arg(exe.join("StdGroup.wxs")), - ); - if built_tools.contains("rust-analyzer") { - builder.run( - Command::new(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rust-analyzer") - .args(&heat_flags) - .arg("-cg") - .arg("RustAnalyzerGroup") - .arg("-dr") - .arg("RustAnalyzer") - .arg("-var") - .arg("var.RustAnalyzerDir") - .arg("-out") - .arg(exe.join("RustAnalyzerGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/remove-duplicates.xsl")), - ); - } - if built_tools.contains("clippy") { - builder.run( - Command::new(&heat) - .current_dir(&exe) - .arg("dir") - .arg("clippy") - .args(&heat_flags) - .arg("-cg") - .arg("ClippyGroup") - .arg("-dr") - .arg("Clippy") - .arg("-var") - .arg("var.ClippyDir") - .arg("-out") - .arg(exe.join("ClippyGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/remove-duplicates.xsl")), - ); - } - if built_tools.contains("rust-demangler") { - builder.run( - Command::new(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rust-demangler") - .args(&heat_flags) - .arg("-cg") - .arg("RustDemanglerGroup") - .arg("-dr") - .arg("RustDemangler") - .arg("-var") - .arg("var.RustDemanglerDir") - .arg("-out") - .arg(exe.join("RustDemanglerGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/remove-duplicates.xsl")), - ); - } - if built_tools.contains("miri") { - builder.run( - Command::new(&heat) - .current_dir(&exe) - .arg("dir") - .arg("miri") - .args(&heat_flags) - .arg("-cg") - .arg("MiriGroup") - .arg("-dr") - .arg("Miri") - .arg("-var") - .arg("var.MiriDir") - .arg("-out") - .arg(exe.join("MiriGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/remove-duplicates.xsl")), - ); - } - builder.run( - Command::new(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rust-analysis") - .args(&heat_flags) - .arg("-cg") - .arg("AnalysisGroup") - .arg("-dr") - .arg("Analysis") - .arg("-var") - .arg("var.AnalysisDir") - .arg("-out") - .arg(exe.join("AnalysisGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/remove-duplicates.xsl")), - ); - if target.ends_with("windows-gnu") { - builder.run( - Command::new(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rust-mingw") - .args(&heat_flags) - .arg("-cg") - .arg("GccGroup") - .arg("-dr") - .arg("Gcc") - .arg("-var") - .arg("var.GccDir") - .arg("-out") - .arg(exe.join("GccGroup.wxs")), - ); - } - - let candle = |input: &Path| { - let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj"); - let arch = if target.contains("x86_64") { "x64" } else { "x86" }; - let mut cmd = Command::new(&candle); - cmd.current_dir(&exe) - .arg("-nologo") - .arg("-dRustcDir=rustc") - .arg("-dCargoDir=cargo") - .arg("-dStdDir=rust-std") - .arg("-dAnalysisDir=rust-analysis") - .arg("-arch") - .arg(&arch) - .arg("-out") - .arg(&output) - .arg(&input); - add_env(builder, &mut cmd, target); - - if built_tools.contains("clippy") { - cmd.arg("-dClippyDir=clippy"); - } - if built_tools.contains("rust-docs") { - cmd.arg("-dDocsDir=rust-docs"); - } - if built_tools.contains("rust-demangler") { - cmd.arg("-dRustDemanglerDir=rust-demangler"); - } - if built_tools.contains("rust-analyzer") { - cmd.arg("-dRustAnalyzerDir=rust-analyzer"); - } - if built_tools.contains("miri") { - cmd.arg("-dMiriDir=miri"); - } - if target.ends_with("windows-gnu") { - cmd.arg("-dGccDir=rust-mingw"); - } - builder.run(&mut cmd); - }; - candle(&xform(&etc.join("msi/rust.wxs"))); - candle(&etc.join("msi/ui.wxs")); - candle(&etc.join("msi/rustwelcomedlg.wxs")); - candle("RustcGroup.wxs".as_ref()); - if built_tools.contains("rust-docs") { - candle("DocsGroup.wxs".as_ref()); - } - candle("CargoGroup.wxs".as_ref()); - candle("StdGroup.wxs".as_ref()); - if built_tools.contains("clippy") { - candle("ClippyGroup.wxs".as_ref()); - } - if built_tools.contains("miri") { - candle("MiriGroup.wxs".as_ref()); - } - if built_tools.contains("rust-demangler") { - candle("RustDemanglerGroup.wxs".as_ref()); - } - if built_tools.contains("rust-analyzer") { - candle("RustAnalyzerGroup.wxs".as_ref()); - } - candle("AnalysisGroup.wxs".as_ref()); - - if target.ends_with("windows-gnu") { - candle("GccGroup.wxs".as_ref()); - } - - builder.create(&exe.join("LICENSE.rtf"), &rtf); - builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644); - builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644); - - builder.info(&format!("building `msi` installer with {light:?}")); - let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple); - let mut cmd = Command::new(&light); - cmd.arg("-nologo") - .arg("-ext") - .arg("WixUIExtension") - .arg("-ext") - .arg("WixUtilExtension") - .arg("-out") - .arg(exe.join(&filename)) - .arg("rust.wixobj") - .arg("ui.wixobj") - .arg("rustwelcomedlg.wixobj") - .arg("RustcGroup.wixobj") - .arg("CargoGroup.wixobj") - .arg("StdGroup.wixobj") - .arg("AnalysisGroup.wixobj") - .current_dir(&exe); - - if built_tools.contains("clippy") { - cmd.arg("ClippyGroup.wixobj"); - } - if built_tools.contains("miri") { - cmd.arg("MiriGroup.wixobj"); - } - if built_tools.contains("rust-analyzer") { - cmd.arg("RustAnalyzerGroup.wixobj"); - } - if built_tools.contains("rust-demangler") { - cmd.arg("RustDemanglerGroup.wixobj"); - } - if built_tools.contains("rust-docs") { - cmd.arg("DocsGroup.wixobj"); - } - - if target.ends_with("windows-gnu") { - cmd.arg("GccGroup.wixobj"); - } - // ICE57 wrongly complains about the shortcuts - cmd.arg("-sice:ICE57"); - - let _time = timeit(builder); - builder.run(&mut cmd); - - if !builder.config.dry_run() { - t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename))); - } - } - } -} - -fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) { - let mut parts = builder.version.split('.'); - cmd.env("CFG_RELEASE_INFO", builder.rust_version()) - .env("CFG_RELEASE_NUM", &builder.version) - .env("CFG_RELEASE", builder.rust_release()) - .env("CFG_VER_MAJOR", parts.next().unwrap()) - .env("CFG_VER_MINOR", parts.next().unwrap()) - .env("CFG_VER_PATCH", parts.next().unwrap()) - .env("CFG_VER_BUILD", "0") // just needed to build - .env("CFG_PACKAGE_VERS", builder.rust_package_vers()) - .env("CFG_PACKAGE_NAME", pkgname(builder, "rust")) - .env("CFG_BUILD", target.triple) - .env("CFG_CHANNEL", &builder.config.channel); - - if target.contains("windows-gnullvm") { - cmd.env("CFG_MINGW", "1").env("CFG_ABI", "LLVM"); - } else if target.contains("windows-gnu") { - cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU"); - } else { - cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC"); - } -} - -fn install_llvm_file(builder: &Builder<'_>, source: &Path, destination: &Path) { - if builder.config.dry_run() { - return; - } - - builder.install(&source, destination, 0o644); -} - -/// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking. -/// -/// Returns whether the files were actually copied. -fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool { - if let Some(config) = builder.config.target_config.get(&target) { - if config.llvm_config.is_some() && !builder.config.llvm_from_ci { - // If the LLVM was externally provided, then we don't currently copy - // artifacts into the sysroot. This is not necessarily the right - // choice (in particular, it will require the LLVM dylib to be in - // the linker's load path at runtime), but the common use case for - // external LLVMs is distribution provided LLVMs, and in that case - // they're usually in the standard search path (e.g., /usr/lib) and - // copying them here is going to cause problems as we may end up - // with the wrong files and isn't what distributions want. - // - // This behavior may be revisited in the future though. - // - // If the LLVM is coming from ourselves (just from CI) though, we - // still want to install it, as it otherwise won't be available. - return false; - } - } - - // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib - // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely - // clear why this is the case, though. llvm-config will emit the versioned - // paths and we don't want those in the sysroot (as we're expecting - // unversioned paths). - if target.contains("apple-darwin") && builder.llvm_link_shared() { - let src_libdir = builder.llvm_out(target).join("lib"); - let llvm_dylib_path = src_libdir.join("libLLVM.dylib"); - if llvm_dylib_path.exists() { - builder.install(&llvm_dylib_path, dst_libdir, 0o644); - } - !builder.config.dry_run() - } else if let Ok(llvm::LlvmResult { llvm_config, .. }) = - llvm::prebuilt_llvm_config(builder, target) - { - let mut cmd = Command::new(llvm_config); - cmd.arg("--libfiles"); - builder.verbose(&format!("running {cmd:?}")); - let files = if builder.config.dry_run() { "".into() } else { output(&mut cmd) }; - let build_llvm_out = &builder.llvm_out(builder.config.build); - let target_llvm_out = &builder.llvm_out(target); - for file in files.trim_end().split(' ') { - // If we're not using a custom LLVM, make sure we package for the target. - let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) { - target_llvm_out.join(relative_path) - } else { - PathBuf::from(file) - }; - install_llvm_file(builder, &file, dst_libdir); - } - !builder.config.dry_run() - } else { - false - } -} - -/// Maybe add libLLVM.so to the target lib-dir for linking. -pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) { - let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib"); - // We do not need to copy LLVM files into the sysroot if it is not - // dynamically linked; it is already included into librustc_llvm - // statically. - if builder.llvm_link_shared() { - maybe_install_llvm(builder, target, &dst_libdir); - } -} - -/// Maybe add libLLVM.so to the runtime lib-dir for rustc itself. -pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) { - let dst_libdir = - sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target })); - // We do not need to copy LLVM files into the sysroot if it is not - // dynamically linked; it is already included into librustc_llvm - // statically. - if builder.llvm_link_shared() { - maybe_install_llvm(builder, target, &dst_libdir); - } -} - -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -pub struct LlvmTools { - pub target: TargetSelection, -} - -impl Step for LlvmTools { - type Output = Option; - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(&run.builder, "llvm-tools"); - // FIXME: allow using the names of the tools themselves? - run.alias("llvm-tools").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(LlvmTools { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let target = self.target; - - /* run only if llvm-config isn't used */ - if let Some(config) = builder.config.target_config.get(&target) { - if let Some(ref _s) = config.llvm_config { - builder.info(&format!("Skipping LlvmTools ({target}): external LLVM")); - return None; - } - } - - builder.ensure(crate::llvm::Llvm { target }); - - let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple); - tarball.set_overlay(OverlayKind::LLVM); - tarball.is_preview(true); - - // Prepare the image directory - let src_bindir = builder.llvm_out(target).join("bin"); - let dst_bindir = format!("lib/rustlib/{}/bin", target.triple); - for tool in LLVM_TOOLS { - let exe = src_bindir.join(exe(tool, target)); - tarball.add_file(&exe, &dst_bindir, 0o755); - } - - // Copy libLLVM.so to the target lib dir as well, so the RPATH like - // `$ORIGIN/../lib` can find it. It may also be used as a dependency - // of `rustc-dev` to support the inherited `-lLLVM` when using the - // compiler libraries. - maybe_install_llvm_target(builder, target, tarball.image_dir()); - - Some(tarball.generate()) - } -} - -// Tarball intended for internal consumption to ease rustc/std development. -// -// Should not be considered stable by end users. -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -pub struct RustDev { - pub target: TargetSelection, -} - -impl Step for RustDev { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("rust-dev") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustDev { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let target = self.target; - - /* run only if llvm-config isn't used */ - if let Some(config) = builder.config.target_config.get(&target) { - if let Some(ref _s) = config.llvm_config { - builder.info(&format!("Skipping RustDev ({target}): external LLVM")); - return None; - } - } - - let mut tarball = Tarball::new(builder, "rust-dev", &target.triple); - tarball.set_overlay(OverlayKind::LLVM); - - builder.ensure(crate::llvm::Llvm { target }); - - // We want to package `lld` to use it with `download-ci-llvm`. - builder.ensure(crate::llvm::Lld { target }); - - let src_bindir = builder.llvm_out(target).join("bin"); - // If updating this list, you likely want to change - // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users - // will not pick up the extra file until LLVM gets bumped. - for bin in &[ - "llvm-config", - "llvm-ar", - "llvm-objdump", - "llvm-profdata", - "llvm-bcanalyzer", - "llvm-cov", - "llvm-dwp", - "llvm-nm", - "llvm-dwarfdump", - "llvm-dis", - "llvm-tblgen", - ] { - tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755); - } - - // We don't build LLD on some platforms, so only add it if it exists - let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target)); - if lld_path.exists() { - tarball.add_file(lld_path, "bin", 0o755); - } - - tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755); - - // Copy the include directory as well; needed mostly to build - // librustc_llvm properly (e.g., llvm-config.h is in here). But also - // just broadly useful to be able to link against the bundled LLVM. - tarball.add_dir(&builder.llvm_out(target).join("include"), "include"); - - // Copy libLLVM.so to the target lib dir as well, so the RPATH like - // `$ORIGIN/../lib` can find it. It may also be used as a dependency - // of `rustc-dev` to support the inherited `-lLLVM` when using the - // compiler libraries. - let dst_libdir = tarball.image_dir().join("lib"); - maybe_install_llvm(builder, target, &dst_libdir); - let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" }; - t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir); - - Some(tarball.generate()) - } -} - -// Tarball intended for internal consumption to ease rustc/std development. -// -// Should not be considered stable by end users. -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -pub struct Bootstrap { - pub target: TargetSelection, -} - -impl Step for Bootstrap { - type Output = Option; - const DEFAULT: bool = false; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("bootstrap") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Bootstrap { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let target = self.target; - - let tarball = Tarball::new(builder, "bootstrap", &target.triple); - - let bootstrap_outdir = &builder.bootstrap_out; - for file in &["bootstrap", "rustc", "rustdoc", "sccache-plus-cl"] { - tarball.add_file(bootstrap_outdir.join(exe(file, target)), "bootstrap/bin", 0o755); - } - - Some(tarball.generate()) - } -} - -/// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the -/// release process to avoid cloning the monorepo and building stuff. -/// -/// Should not be considered stable by end users. -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -pub struct BuildManifest { - pub target: TargetSelection, -} - -impl Step for BuildManifest { - type Output = GeneratedTarball; - const DEFAULT: bool = false; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("build-manifest") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(BuildManifest { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) -> GeneratedTarball { - let build_manifest = builder.tool_exe(Tool::BuildManifest); - - let tarball = Tarball::new(builder, "build-manifest", &self.target.triple); - tarball.add_file(&build_manifest, "bin", 0o755); - tarball.generate() - } -} - -/// Tarball containing artifacts necessary to reproduce the build of rustc. -/// -/// Currently this is the PGO profile data. -/// -/// Should not be considered stable by end users. -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -pub struct ReproducibleArtifacts { - pub target: TargetSelection, -} - -impl Step for ReproducibleArtifacts { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("reproducible-artifacts") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(ReproducibleArtifacts { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - let mut added_anything = false; - let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple); - if let Some(path) = builder.config.rust_profile_use.as_ref() { - tarball.add_file(path, ".", 0o644); - added_anything = true; - } - if let Some(path) = builder.config.llvm_profile_use.as_ref() { - tarball.add_file(path, ".", 0o644); - added_anything = true; - } - for profile in &builder.config.reproducible_artifacts { - tarball.add_file(profile, ".", 0o644); - added_anything = true; - } - if added_anything { Some(tarball.generate()) } else { None } - } -} diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs deleted file mode 100644 index 505f06ed1..000000000 --- a/src/bootstrap/doc.rs +++ /dev/null @@ -1,1085 +0,0 @@ -//! Documentation generation for rustbuilder. -//! -//! This module implements generation for all bits and pieces of documentation -//! for the Rust project. This notably includes suites like the rust book, the -//! nomicon, rust by example, standalone documentation, etc. -//! -//! Everything here is basically just a shim around calling either `rustbook` or -//! `rustdoc`. - -use std::fs; -use std::path::{Path, PathBuf}; - -use crate::builder::crate_description; -use crate::builder::{Alias, Builder, Compiler, Kind, RunConfig, ShouldRun, Step}; -use crate::cache::{Interned, INTERNER}; -use crate::compile; -use crate::config::{Config, TargetSelection}; -use crate::tool::{self, prepare_tool_cargo, SourceType, Tool}; -use crate::util::{dir_is_empty, symlink_dir, t, up_to_date}; -use crate::Mode; - -macro_rules! submodule_helper { - ($path:expr, submodule) => { - $path - }; - ($path:expr, submodule = $submodule:literal) => { - $submodule - }; -} - -macro_rules! book { - ($($name:ident, $path:expr, $book_name:expr $(, submodule $(= $submodule:literal)? )? ;)+) => { - $( - #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] - pub struct $name { - target: TargetSelection, - } - - impl Step for $name { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path($path).default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure($name { - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) { - $( - let path = Path::new(submodule_helper!( $path, submodule $( = $submodule )? )); - builder.update_submodule(&path); - )? - builder.ensure(RustbookSrc { - target: self.target, - name: INTERNER.intern_str($book_name), - src: INTERNER.intern_path(builder.src.join($path)), - parent: Some(self), - }) - } - } - )+ - } -} - -// NOTE: When adding a book here, make sure to ALSO build the book by -// adding a build step in `src/bootstrap/builder.rs`! -// NOTE: Make sure to add the corresponding submodule when adding a new book. -// FIXME: Make checking for a submodule automatic somehow (maybe by having a list of all submodules -// and checking against it?). -book!( - CargoBook, "src/tools/cargo/src/doc", "cargo", submodule = "src/tools/cargo"; - ClippyBook, "src/tools/clippy/book", "clippy"; - EditionGuide, "src/doc/edition-guide", "edition-guide", submodule; - EmbeddedBook, "src/doc/embedded-book", "embedded-book", submodule; - Nomicon, "src/doc/nomicon", "nomicon", submodule; - Reference, "src/doc/reference", "reference", submodule; - RustByExample, "src/doc/rust-by-example", "rust-by-example", submodule; - RustdocBook, "src/doc/rustdoc", "rustdoc"; - StyleGuide, "src/doc/style-guide", "style-guide"; -); - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct UnstableBook { - target: TargetSelection, -} - -impl Step for UnstableBook { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/doc/unstable-book").default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(UnstableBook { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - builder.ensure(UnstableBookGen { target: self.target }); - builder.ensure(RustbookSrc { - target: self.target, - name: INTERNER.intern_str("unstable-book"), - src: INTERNER.intern_path(builder.md_doc_out(self.target).join("unstable-book")), - parent: Some(self), - }) - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -struct RustbookSrc { - target: TargetSelection, - name: Interned, - src: Interned, - parent: Option

, -} - -impl Step for RustbookSrc

{ - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Invoke `rustbook` for `target` for the doc book `name` from the `src` path. - /// - /// This will not actually generate any documentation if the documentation has - /// already been generated. - fn run(self, builder: &Builder<'_>) { - let target = self.target; - let name = self.name; - let src = self.src; - let out = builder.doc_out(target); - t!(fs::create_dir_all(&out)); - - let out = out.join(name); - let index = out.join("index.html"); - let rustbook = builder.tool_exe(Tool::Rustbook); - let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook); - - if !builder.config.dry_run() && !(up_to_date(&src, &index) || up_to_date(&rustbook, &index)) - { - builder.info(&format!("Rustbook ({target}) - {name}")); - let _ = fs::remove_dir_all(&out); - - builder.run(rustbook_cmd.arg("build").arg(&src).arg("-d").arg(out)); - } - - if self.parent.is_some() { - builder.maybe_open_in_browser::

(index) - } - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct TheBook { - compiler: Compiler, - target: TargetSelection, -} - -impl Step for TheBook { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/doc/book").default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(TheBook { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - /// Builds the book and associated stuff. - /// - /// We need to build: - /// - /// * Book - /// * Older edition redirects - /// * Version info and CSS - /// * Index page - /// * Redirect pages - fn run(self, builder: &Builder<'_>) { - let relative_path = Path::new("src").join("doc").join("book"); - builder.update_submodule(&relative_path); - - let compiler = self.compiler; - let target = self.target; - - let absolute_path = builder.src.join(&relative_path); - let redirect_path = absolute_path.join("redirects"); - if !absolute_path.exists() - || !redirect_path.exists() - || dir_is_empty(&absolute_path) - || dir_is_empty(&redirect_path) - { - eprintln!("Please checkout submodule: {}", relative_path.display()); - crate::exit!(1); - } - // build book - builder.ensure(RustbookSrc { - target, - name: INTERNER.intern_str("book"), - src: INTERNER.intern_path(absolute_path.clone()), - parent: Some(self), - }); - - // building older edition redirects - for edition in &["first-edition", "second-edition", "2018-edition"] { - builder.ensure(RustbookSrc { - target, - name: INTERNER.intern_string(format!("book/{edition}")), - src: INTERNER.intern_path(absolute_path.join(edition)), - // There should only be one book that is marked as the parent for each target, so - // treat the other editions as not having a parent. - parent: Option::::None, - }); - } - - // build the version info page and CSS - let shared_assets = builder.ensure(SharedAssets { target }); - - // build the command first so we don't nest GHA groups - builder.rustdoc_cmd(compiler); - - // build the redirect pages - let _guard = builder.msg_doc(compiler, "book redirect pages", target); - for file in t!(fs::read_dir(redirect_path)) { - let file = t!(file); - let path = file.path(); - let path = path.to_str().unwrap(); - - invoke_rustdoc(builder, compiler, &shared_assets, target, path); - } - } -} - -fn invoke_rustdoc( - builder: &Builder<'_>, - compiler: Compiler, - shared_assets: &SharedAssetsPaths, - target: TargetSelection, - markdown: &str, -) { - let out = builder.doc_out(target); - - let path = builder.src.join("src/doc").join(markdown); - - let header = builder.src.join("src/doc/redirect.inc"); - let footer = builder.src.join("src/doc/footer.inc"); - - let mut cmd = builder.rustdoc_cmd(compiler); - - let out = out.join("book"); - - cmd.arg("--html-after-content") - .arg(&footer) - .arg("--html-before-content") - .arg(&shared_assets.version_info) - .arg("--html-in-header") - .arg(&header) - .arg("--markdown-no-toc") - .arg("--markdown-playground-url") - .arg("https://play.rust-lang.org/") - .arg("-o") - .arg(&out) - .arg(&path) - .arg("--markdown-css") - .arg("../rust.css"); - - if !builder.config.docs_minification { - cmd.arg("-Z").arg("unstable-options").arg("--disable-minification"); - } - - builder.run(&mut cmd); -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Standalone { - compiler: Compiler, - target: TargetSelection, -} - -impl Step for Standalone { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/doc").alias("standalone").default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Standalone { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - /// Generates all standalone documentation as compiled by the rustdoc in `stage` - /// for the `target` into `out`. - /// - /// This will list all of `src/doc` looking for markdown files and appropriately - /// perform transformations like substituting `VERSION`, `SHORT_HASH`, and - /// `STAMP` along with providing the various header/footer HTML we've customized. - /// - /// In the end, this is just a glorified wrapper around rustdoc! - fn run(self, builder: &Builder<'_>) { - let target = self.target; - let compiler = self.compiler; - let _guard = builder.msg_doc(compiler, "standalone", target); - let out = builder.doc_out(target); - t!(fs::create_dir_all(&out)); - - let version_info = builder.ensure(SharedAssets { target: self.target }).version_info; - - let favicon = builder.src.join("src/doc/favicon.inc"); - let footer = builder.src.join("src/doc/footer.inc"); - let full_toc = builder.src.join("src/doc/full-toc.inc"); - - for file in t!(fs::read_dir(builder.src.join("src/doc"))) { - let file = t!(file); - let path = file.path(); - let filename = path.file_name().unwrap().to_str().unwrap(); - if !filename.ends_with(".md") || filename == "README.md" { - continue; - } - - let html = out.join(filename).with_extension("html"); - let rustdoc = builder.rustdoc(compiler); - if up_to_date(&path, &html) - && up_to_date(&footer, &html) - && up_to_date(&favicon, &html) - && up_to_date(&full_toc, &html) - && (builder.config.dry_run() || up_to_date(&version_info, &html)) - && (builder.config.dry_run() || up_to_date(&rustdoc, &html)) - { - continue; - } - - let mut cmd = builder.rustdoc_cmd(compiler); - // Needed for --index-page flag - cmd.arg("-Z").arg("unstable-options"); - - cmd.arg("--html-after-content") - .arg(&footer) - .arg("--html-before-content") - .arg(&version_info) - .arg("--html-in-header") - .arg(&favicon) - .arg("--markdown-no-toc") - .arg("--index-page") - .arg(&builder.src.join("src/doc/index.md")) - .arg("--markdown-playground-url") - .arg("https://play.rust-lang.org/") - .arg("-o") - .arg(&out) - .arg(&path); - - if !builder.config.docs_minification { - cmd.arg("--disable-minification"); - } - - if filename == "not_found.md" { - cmd.arg("--markdown-css").arg("https://doc.rust-lang.org/rust.css"); - } else { - cmd.arg("--markdown-css").arg("rust.css"); - } - builder.run(&mut cmd); - } - - // We open doc/index.html as the default if invoked as `x.py doc --open` - // with no particular explicit doc requested (e.g. library/core). - if builder.paths.is_empty() || builder.was_invoked_explicitly::(Kind::Doc) { - let index = out.join("index.html"); - builder.open_in_browser(&index); - } - } -} - -#[derive(Debug, Clone)] -pub struct SharedAssetsPaths { - pub version_info: PathBuf, -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct SharedAssets { - target: TargetSelection, -} - -impl Step for SharedAssets { - type Output = SharedAssetsPaths; - const DEFAULT: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - // Other tasks depend on this, no need to execute it on its own - run.never() - } - - // Generate shared resources used by other pieces of documentation. - fn run(self, builder: &Builder<'_>) -> Self::Output { - let out = builder.doc_out(self.target); - - let version_input = builder.src.join("src").join("doc").join("version_info.html.template"); - let version_info = out.join("version_info.html"); - if !builder.config.dry_run() && !up_to_date(&version_input, &version_info) { - let info = t!(fs::read_to_string(&version_input)) - .replace("VERSION", &builder.rust_release()) - .replace("SHORT_HASH", builder.rust_info().sha_short().unwrap_or("")) - .replace("STAMP", builder.rust_info().sha().unwrap_or("")); - t!(fs::write(&version_info, &info)); - } - - builder.copy(&builder.src.join("src").join("doc").join("rust.css"), &out.join("rust.css")); - - SharedAssetsPaths { version_info } - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] -pub struct Std { - pub stage: u32, - pub target: TargetSelection, - pub format: DocumentationFormat, - crates: Interned>, -} - -impl Std { - pub(crate) fn new( - stage: u32, - target: TargetSelection, - builder: &Builder<'_>, - format: DocumentationFormat, - ) -> Self { - let crates = builder - .in_tree_crates("sysroot", Some(target)) - .into_iter() - .map(|krate| krate.name.to_string()) - .collect(); - Std { stage, target, format, crates: INTERNER.intern_list(crates) } - } -} - -impl Step for Std { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.crate_or_deps("sysroot").path("library").default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Std { - stage: run.builder.top_stage, - target: run.target, - format: if run.builder.config.cmd.json() { - DocumentationFormat::JSON - } else { - DocumentationFormat::HTML - }, - crates: run.make_run_crates(Alias::Library), - }); - } - - /// Compile all standard library documentation. - /// - /// This will generate all documentation for the standard library and its - /// dependencies. This is largely just a wrapper around `cargo doc`. - fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let target = self.target; - let out = match self.format { - DocumentationFormat::HTML => builder.doc_out(target), - DocumentationFormat::JSON => builder.json_doc_out(target), - }; - - t!(fs::create_dir_all(&out)); - - if self.format == DocumentationFormat::HTML { - builder.ensure(SharedAssets { target: self.target }); - } - - let index_page = builder - .src - .join("src/doc/index.md") - .into_os_string() - .into_string() - .expect("non-utf8 paths are unsupported"); - let mut extra_args = match self.format { - DocumentationFormat::HTML => { - vec!["--markdown-css", "rust.css", "--markdown-no-toc", "--index-page", &index_page] - } - DocumentationFormat::JSON => vec!["--output-format", "json"], - }; - - if !builder.config.docs_minification { - extra_args.push("--disable-minification"); - } - - doc_std(builder, self.format, stage, target, &out, &extra_args, &self.crates); - - // Don't open if the format is json - if let DocumentationFormat::JSON = self.format { - return; - } - - if builder.paths.iter().any(|path| path.ends_with("library")) { - // For `x.py doc library --open`, open `std` by default. - let index = out.join("std").join("index.html"); - builder.open_in_browser(index); - } else { - for requested_crate in &*self.crates { - if STD_PUBLIC_CRATES.iter().any(|&k| k == requested_crate) { - let index = out.join(requested_crate).join("index.html"); - builder.open_in_browser(index); - break; - } - } - } - } -} - -/// Name of the crates that are visible to consumers of the standard library. -/// Documentation for internal crates is handled by the rustc step, so internal crates will show -/// up there. -/// -/// Order here is important! -/// Crates need to be processed starting from the leaves, otherwise rustdoc will not -/// create correct links between crates because rustdoc depends on the -/// existence of the output directories to know if it should be a local -/// or remote link. -const STD_PUBLIC_CRATES: [&str; 5] = ["core", "alloc", "std", "proc_macro", "test"]; - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] -pub enum DocumentationFormat { - HTML, - JSON, -} - -impl DocumentationFormat { - fn as_str(&self) -> &str { - match self { - DocumentationFormat::HTML => "HTML", - DocumentationFormat::JSON => "JSON", - } - } -} - -/// Build the documentation for public standard library crates. -fn doc_std( - builder: &Builder<'_>, - format: DocumentationFormat, - stage: u32, - target: TargetSelection, - out: &Path, - extra_args: &[&str], - requested_crates: &[String], -) { - if builder.no_std(target) == Some(true) { - panic!( - "building std documentation for no_std target {target} is not supported\n\ - Set `docs = false` in the config to disable documentation, or pass `--skip library`." - ); - } - - let compiler = builder.compiler(stage, builder.config.build); - - let target_doc_dir_name = if format == DocumentationFormat::JSON { "json-doc" } else { "doc" }; - let target_dir = - builder.stage_out(compiler, Mode::Std).join(target.triple).join(target_doc_dir_name); - - // This is directory where the compiler will place the output of the command. - // We will then copy the files from this directory into the final `out` directory, the specified - // as a function parameter. - let out_dir = target_dir.join(target.triple).join("doc"); - - let mut cargo = builder.cargo(compiler, Mode::Std, SourceType::InTree, target, "doc"); - compile::std_cargo(builder, target, compiler.stage, &mut cargo); - cargo - .arg("--no-deps") - .arg("--target-dir") - .arg(&*target_dir.to_string_lossy()) - .arg("-Zskip-rustdoc-fingerprint") - .rustdocflag("-Z") - .rustdocflag("unstable-options") - .rustdocflag("--resource-suffix") - .rustdocflag(&builder.version); - for arg in extra_args { - cargo.rustdocflag(arg); - } - - if builder.config.library_docs_private_items { - cargo.rustdocflag("--document-private-items").rustdocflag("--document-hidden-items"); - } - - for krate in requested_crates { - if krate == "sysroot" { - // The sysroot crate is an implementation detail, don't include it in public docs. - continue; - } - cargo.arg("-p").arg(krate); - } - - let description = - format!("library{} in {} format", crate_description(&requested_crates), format.as_str()); - let _guard = builder.msg_doc(compiler, &description, target); - - builder.run(&mut cargo.into()); - builder.cp_r(&out_dir, &out); -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Rustc { - pub stage: u32, - pub target: TargetSelection, - crates: Interned>, -} - -impl Rustc { - pub(crate) fn new(stage: u32, target: TargetSelection, builder: &Builder<'_>) -> Self { - let crates = builder - .in_tree_crates("rustc-main", Some(target)) - .into_iter() - .map(|krate| krate.name.to_string()) - .collect(); - Self { stage, target, crates: INTERNER.intern_list(crates) } - } -} - -impl Step for Rustc { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.crate_or_deps("rustc-main") - .path("compiler") - .default_condition(builder.config.compiler_docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Rustc { - stage: run.builder.top_stage, - target: run.target, - crates: run.make_run_crates(Alias::Compiler), - }); - } - - /// Generates compiler documentation. - /// - /// This will generate all documentation for compiler and dependencies. - /// Compiler documentation is distributed separately, so we make sure - /// we do not merge it with the other documentation from std, test and - /// proc_macros. This is largely just a wrapper around `cargo doc`. - fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let target = self.target; - - // This is the intended out directory for compiler documentation. - let out = builder.compiler_doc_out(target); - t!(fs::create_dir_all(&out)); - - // Build the standard library, so that proc-macros can use it. - // (Normally, only the metadata would be necessary, but proc-macros are special since they run at compile-time.) - let compiler = builder.compiler(stage, builder.config.build); - builder.ensure(compile::Std::new(compiler, builder.config.build)); - - let _guard = builder.msg_sysroot_tool( - Kind::Doc, - stage, - &format!("compiler{}", crate_description(&self.crates)), - compiler.host, - target, - ); - - // This uses a shared directory so that librustdoc documentation gets - // correctly built and merged with the rustc documentation. This is - // needed because rustdoc is built in a different directory from - // rustc. rustdoc needs to be able to see everything, for example when - // merging the search index, or generating local (relative) links. - let out_dir = builder.stage_out(compiler, Mode::Rustc).join(target.triple).join("doc"); - t!(fs::create_dir_all(out_dir.parent().unwrap())); - symlink_dir_force(&builder.config, &out, &out_dir); - // Cargo puts proc macros in `target/doc` even if you pass `--target` - // explicitly (https://github.com/rust-lang/cargo/issues/7677). - let proc_macro_out_dir = builder.stage_out(compiler, Mode::Rustc).join("doc"); - symlink_dir_force(&builder.config, &out, &proc_macro_out_dir); - - // Build cargo command. - let mut cargo = builder.cargo(compiler, Mode::Rustc, SourceType::InTree, target, "doc"); - cargo.rustdocflag("--document-private-items"); - // Since we always pass --document-private-items, there's no need to warn about linking to private items. - cargo.rustdocflag("-Arustdoc::private-intra-doc-links"); - cargo.rustdocflag("--enable-index-page"); - cargo.rustdocflag("-Zunstable-options"); - cargo.rustdocflag("-Znormalize-docs"); - cargo.rustdocflag("--show-type-layout"); - cargo.rustdocflag("--generate-link-to-definition"); - compile::rustc_cargo(builder, &mut cargo, target, compiler.stage); - cargo.arg("-Zunstable-options"); - cargo.arg("-Zskip-rustdoc-fingerprint"); - - // Only include compiler crates, no dependencies of those, such as `libc`. - // Do link to dependencies on `docs.rs` however using `rustdoc-map`. - cargo.arg("--no-deps"); - cargo.arg("-Zrustdoc-map"); - - // FIXME: `-Zrustdoc-map` does not yet correctly work for transitive dependencies, - // once this is no longer an issue the special case for `ena` can be removed. - cargo.rustdocflag("--extern-html-root-url"); - cargo.rustdocflag("ena=https://docs.rs/ena/latest/"); - - let mut to_open = None; - - for krate in &*self.crates { - // Create all crate output directories first to make sure rustdoc uses - // relative links. - // FIXME: Cargo should probably do this itself. - let dir_name = krate.replace("-", "_"); - t!(fs::create_dir_all(out_dir.join(&*dir_name))); - cargo.arg("-p").arg(krate); - if to_open.is_none() { - to_open = Some(dir_name); - } - } - - builder.run(&mut cargo.into()); - - if builder.paths.iter().any(|path| path.ends_with("compiler")) { - // For `x.py doc compiler --open`, open `rustc_middle` by default. - let index = out.join("rustc_middle").join("index.html"); - builder.open_in_browser(index); - } else if let Some(krate) = to_open { - // Let's open the first crate documentation page: - let index = out.join(krate).join("index.html"); - builder.open_in_browser(index); - } - } -} - -macro_rules! tool_doc { - ( - $tool: ident, - $should_run: literal, - $path: literal, - $(rustc_tool = $rustc_tool:literal, )? - $(in_tree = $in_tree:literal, )? - [$($extra_arg: literal),+ $(,)?] - $(,)? - ) => { - #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] - pub struct $tool { - target: TargetSelection, - } - - impl Step for $tool { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.crate_or_deps($should_run).default_condition(builder.config.compiler_docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure($tool { target: run.target }); - } - - /// Generates compiler documentation. - /// - /// This will generate all documentation for compiler and dependencies. - /// Compiler documentation is distributed separately, so we make sure - /// we do not merge it with the other documentation from std, test and - /// proc_macros. This is largely just a wrapper around `cargo doc`. - fn run(self, builder: &Builder<'_>) { - let stage = builder.top_stage; - let target = self.target; - - // This is the intended out directory for compiler documentation. - let out = builder.compiler_doc_out(target); - t!(fs::create_dir_all(&out)); - - let compiler = builder.compiler(stage, builder.config.build); - builder.ensure(compile::Std::new(compiler, target)); - - if true $(&& $rustc_tool)? { - // Build rustc docs so that we generate relative links. - builder.ensure(Rustc::new(stage, target, builder)); - - // Rustdoc needs the rustc sysroot available to build. - // FIXME: is there a way to only ensure `check::Rustc` here? Last time I tried it failed - // with strange errors, but only on a full bors test ... - builder.ensure(compile::Rustc::new(compiler, target)); - } - - let source_type = if true $(&& $in_tree)? { - SourceType::InTree - } else { - SourceType::Submodule - }; - - // Symlink compiler docs to the output directory of rustdoc documentation. - let out_dirs = [ - builder.stage_out(compiler, Mode::ToolRustc).join(target.triple).join("doc"), - // Cargo uses a different directory for proc macros. - builder.stage_out(compiler, Mode::ToolRustc).join("doc"), - ]; - for out_dir in out_dirs { - t!(fs::create_dir_all(&out_dir)); - symlink_dir_force(&builder.config, &out, &out_dir); - } - - // Build cargo command. - let mut cargo = prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - target, - "doc", - $path, - source_type, - &[], - ); - - cargo.arg("-Zskip-rustdoc-fingerprint"); - // Only include compiler crates, no dependencies of those, such as `libc`. - cargo.arg("--no-deps"); - - $( - cargo.arg($extra_arg); - )+ - - cargo.rustdocflag("--document-private-items"); - // Since we always pass --document-private-items, there's no need to warn about linking to private items. - cargo.rustdocflag("-Arustdoc::private-intra-doc-links"); - cargo.rustdocflag("--enable-index-page"); - cargo.rustdocflag("--show-type-layout"); - cargo.rustdocflag("--generate-link-to-definition"); - cargo.rustdocflag("-Zunstable-options"); - - let _guard = builder.msg_doc(compiler, stringify!($tool).to_lowercase(), target); - builder.run(&mut cargo.into()); - } - } - } -} - -tool_doc!( - Rustdoc, - "rustdoc-tool", - "src/tools/rustdoc", - ["-p", "rustdoc", "-p", "rustdoc-json-types"] -); -tool_doc!( - Rustfmt, - "rustfmt-nightly", - "src/tools/rustfmt", - ["-p", "rustfmt-nightly", "-p", "rustfmt-config_proc_macro"], -); -tool_doc!(Clippy, "clippy", "src/tools/clippy", ["-p", "clippy_utils"]); -tool_doc!(Miri, "miri", "src/tools/miri", ["-p", "miri"]); -tool_doc!( - Cargo, - "cargo", - "src/tools/cargo", - rustc_tool = false, - in_tree = false, - [ - "-p", - "cargo", - "-p", - "cargo-platform", - "-p", - "cargo-util", - "-p", - "crates-io", - "-p", - "cargo-test-macro", - "-p", - "cargo-test-support", - "-p", - "cargo-credential", - "-p", - "mdman", - // FIXME: this trips a license check in tidy. - // "-p", - // "resolver-tests", - ] -); -tool_doc!(Tidy, "tidy", "src/tools/tidy", rustc_tool = false, ["-p", "tidy"]); -tool_doc!( - Bootstrap, - "bootstrap", - "src/bootstrap", - rustc_tool = false, - ["--lib", "-p", "bootstrap"] -); - -#[derive(Ord, PartialOrd, Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct ErrorIndex { - pub target: TargetSelection, -} - -impl Step for ErrorIndex { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/tools/error_index_generator").default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - let target = run.target; - run.builder.ensure(ErrorIndex { target }); - } - - /// Generates the HTML rendered error-index by running the - /// `error_index_generator` tool. - fn run(self, builder: &Builder<'_>) { - builder.info(&format!("Documenting error index ({})", self.target)); - let out = builder.doc_out(self.target); - t!(fs::create_dir_all(&out)); - let mut index = tool::ErrorIndex::command(builder); - index.arg("html"); - index.arg(out); - index.arg(&builder.version); - - builder.run(&mut index); - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct UnstableBookGen { - target: TargetSelection, -} - -impl Step for UnstableBookGen { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/tools/unstable-book-gen").default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(UnstableBookGen { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let target = self.target; - - builder.info(&format!("Generating unstable book md files ({target})")); - let out = builder.md_doc_out(target).join("unstable-book"); - builder.create_dir(&out); - builder.remove_dir(&out); - let mut cmd = builder.tool_cmd(Tool::UnstableBookGen); - cmd.arg(builder.src.join("library")); - cmd.arg(builder.src.join("compiler")); - cmd.arg(builder.src.join("src")); - cmd.arg(out); - - builder.run(&mut cmd); - } -} - -fn symlink_dir_force(config: &Config, original: &Path, link: &Path) { - if config.dry_run() { - return; - } - if let Ok(m) = fs::symlink_metadata(link) { - if m.file_type().is_dir() { - t!(fs::remove_dir_all(link)); - } else { - // handle directory junctions on windows by falling back to - // `remove_dir`. - t!(fs::remove_file(link).or_else(|_| fs::remove_dir(link))); - } - } - - t!( - symlink_dir(config, original, link), - format!("failed to create link from {} -> {}", link.display(), original.display()) - ); -} - -#[derive(Ord, PartialOrd, Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct RustcBook { - pub compiler: Compiler, - pub target: TargetSelection, - pub validate: bool, -} - -impl Step for RustcBook { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/doc/rustc").default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustcBook { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - validate: false, - }); - } - - /// Builds the rustc book. - /// - /// The lints are auto-generated by a tool, and then merged into the book - /// in the "md-doc" directory in the build output directory. Then - /// "rustbook" is used to convert it to HTML. - fn run(self, builder: &Builder<'_>) { - let out_base = builder.md_doc_out(self.target).join("rustc"); - t!(fs::create_dir_all(&out_base)); - let out_listing = out_base.join("src/lints"); - builder.cp_r(&builder.src.join("src/doc/rustc"), &out_base); - builder.info(&format!("Generating lint docs ({})", self.target)); - - let rustc = builder.rustc(self.compiler); - // The tool runs `rustc` for extracting output examples, so it needs a - // functional sysroot. - builder.ensure(compile::Std::new(self.compiler, self.target)); - let mut cmd = builder.tool_cmd(Tool::LintDocs); - cmd.arg("--src"); - cmd.arg(builder.src.join("compiler")); - cmd.arg("--out"); - cmd.arg(&out_listing); - cmd.arg("--rustc"); - cmd.arg(&rustc); - cmd.arg("--rustc-target").arg(&self.target.rustc_target_arg()); - if builder.is_verbose() { - cmd.arg("--verbose"); - } - if self.validate { - cmd.arg("--validate"); - } - // We need to validate nightly features, even on the stable channel. - // Set this unconditionally as the stage0 compiler may be being used to - // document. - cmd.env("RUSTC_BOOTSTRAP", "1"); - - // If the lib directories are in an unusual location (changed in - // config.toml), then this needs to explicitly update the dylib search - // path. - builder.add_rustc_lib_path(self.compiler, &mut cmd); - let doc_generator_guard = builder.msg( - Kind::Run, - self.compiler.stage, - "lint-docs", - self.compiler.host, - self.target, - ); - builder.run(&mut cmd); - drop(doc_generator_guard); - - // Run rustbook/mdbook to generate the HTML pages. - builder.ensure(RustbookSrc { - target: self.target, - name: INTERNER.intern_str("rustc"), - src: INTERNER.intern_path(out_base), - parent: Some(self), - }); - } -} diff --git a/src/bootstrap/download-ci-llvm-stamp b/src/bootstrap/download-ci-llvm-stamp index ffc380579..9998fe2f5 100644 --- a/src/bootstrap/download-ci-llvm-stamp +++ b/src/bootstrap/download-ci-llvm-stamp @@ -1,4 +1,4 @@ Change this file to make users of the `download-ci-llvm` configuration download a new version of LLVM from CI, even if the LLVM submodule hasn’t changed. -Last change is for: https://github.com/rust-lang/rust/pull/113996 +Last change is for: https://github.com/rust-lang/rust/pull/116881 diff --git a/src/bootstrap/download.rs b/src/bootstrap/download.rs deleted file mode 100644 index 8e9614ec8..000000000 --- a/src/bootstrap/download.rs +++ /dev/null @@ -1,686 +0,0 @@ -use std::{ - env, - ffi::{OsStr, OsString}, - fs::{self, File}, - io::{BufRead, BufReader, BufWriter, ErrorKind, Write}, - path::{Path, PathBuf}, - process::{Command, Stdio}, -}; - -use build_helper::ci::CiEnv; -use once_cell::sync::OnceCell; -use xz2::bufread::XzDecoder; - -use crate::{ - config::RustfmtMetadata, - llvm::detect_llvm_sha, - t, - util::{check_run, exe, program_out_of_date}, - Config, -}; - -static SHOULD_FIX_BINS_AND_DYLIBS: OnceCell = OnceCell::new(); - -/// `Config::try_run` wrapper for this module to avoid warnings on `try_run`, since we don't have access to a `builder` yet. -fn try_run(config: &Config, cmd: &mut Command) -> Result<(), ()> { - #[allow(deprecated)] - config.try_run(cmd) -} - -/// Generic helpers that are useful anywhere in bootstrap. -impl Config { - pub fn is_verbose(&self) -> bool { - self.verbose > 0 - } - - pub(crate) fn create(&self, path: &Path, s: &str) { - if self.dry_run() { - return; - } - t!(fs::write(path, s)); - } - - pub(crate) fn remove(&self, f: &Path) { - if self.dry_run() { - return; - } - fs::remove_file(f).unwrap_or_else(|_| panic!("failed to remove {:?}", f)); - } - - /// Create a temporary directory in `out` and return its path. - /// - /// NOTE: this temporary directory is shared between all steps; - /// if you need an empty directory, create a new subdirectory inside it. - pub(crate) fn tempdir(&self) -> PathBuf { - let tmp = self.out.join("tmp"); - t!(fs::create_dir_all(&tmp)); - tmp - } - - /// Runs a command, printing out nice contextual information if it fails. - /// Returns false if do not execute at all, otherwise returns its - /// `status.success()`. - pub(crate) fn check_run(&self, cmd: &mut Command) -> bool { - if self.dry_run() { - return true; - } - self.verbose(&format!("running: {cmd:?}")); - check_run(cmd, self.is_verbose()) - } - - /// Whether or not `fix_bin_or_dylib` needs to be run; can only be true - /// on NixOS - fn should_fix_bins_and_dylibs(&self) -> bool { - let val = *SHOULD_FIX_BINS_AND_DYLIBS.get_or_init(|| { - match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() { - Err(_) => return false, - Ok(output) if !output.status.success() => return false, - Ok(output) => { - let mut os_name = output.stdout; - if os_name.last() == Some(&b'\n') { - os_name.pop(); - } - if os_name != b"Linux" { - return false; - } - } - } - - // If the user has asked binaries to be patched for Nix, then - // don't check for NixOS or `/lib`. - // NOTE: this intentionally comes after the Linux check: - // - patchelf only works with ELF files, so no need to run it on Mac or Windows - // - On other Unix systems, there is no stable syscall interface, so Nix doesn't manage the global libc. - if let Some(explicit_value) = self.patch_binaries_for_nix { - return explicit_value; - } - - // Use `/etc/os-release` instead of `/etc/NIXOS`. - // The latter one does not exist on NixOS when using tmpfs as root. - let is_nixos = match File::open("/etc/os-release") { - Err(e) if e.kind() == ErrorKind::NotFound => false, - Err(e) => panic!("failed to access /etc/os-release: {}", e), - Ok(os_release) => BufReader::new(os_release).lines().any(|l| { - let l = l.expect("reading /etc/os-release"); - matches!(l.trim(), "ID=nixos" | "ID='nixos'" | "ID=\"nixos\"") - }), - }; - if !is_nixos { - let in_nix_shell = env::var("IN_NIX_SHELL"); - if let Ok(in_nix_shell) = in_nix_shell { - eprintln!( - "The IN_NIX_SHELL environment variable is `{in_nix_shell}`; \ - you may need to set `patch-binaries-for-nix=true` in config.toml" - ); - } - } - is_nixos - }); - if val { - eprintln!("info: You seem to be using Nix."); - } - val - } - - /// Modifies the interpreter section of 'fname' to fix the dynamic linker, - /// or the RPATH section, to fix the dynamic library search path - /// - /// This is only required on NixOS and uses the PatchELF utility to - /// change the interpreter/RPATH of ELF executables. - /// - /// Please see for more information - fn fix_bin_or_dylib(&self, fname: &Path) { - assert_eq!(SHOULD_FIX_BINS_AND_DYLIBS.get(), Some(&true)); - println!("attempting to patch {}", fname.display()); - - // Only build `.nix-deps` once. - static NIX_DEPS_DIR: OnceCell = OnceCell::new(); - let mut nix_build_succeeded = true; - let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| { - // Run `nix-build` to "build" each dependency (which will likely reuse - // the existing `/nix/store` copy, or at most download a pre-built copy). - // - // Importantly, we create a gc-root called `.nix-deps` in the `build/` - // directory, but still reference the actual `/nix/store` path in the rpath - // as it makes it significantly more robust against changes to the location of - // the `.nix-deps` location. - // - // bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`). - // zlib: Needed as a system dependency of `libLLVM-*.so`. - // patchelf: Needed for patching ELF binaries (see doc comment above). - let nix_deps_dir = self.out.join(".nix-deps"); - const NIX_EXPR: &str = " - with (import {}); - symlinkJoin { - name = \"rust-stage0-dependencies\"; - paths = [ - zlib - patchelf - stdenv.cc.bintools - ]; - } - "; - nix_build_succeeded = try_run( - self, - Command::new("nix-build").args(&[ - Path::new("-E"), - Path::new(NIX_EXPR), - Path::new("-o"), - &nix_deps_dir, - ]), - ) - .is_ok(); - nix_deps_dir - }); - if !nix_build_succeeded { - return; - } - - let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf")); - let rpath_entries = { - // ORIGIN is a relative default, all binary and dynamic libraries we ship - // appear to have this (even when `../lib` is redundant). - // NOTE: there are only two paths here, delimited by a `:` - let mut entries = OsString::from("$ORIGIN/../lib:"); - entries.push(t!(fs::canonicalize(nix_deps_dir)).join("lib")); - entries - }; - patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]); - if !fname.extension().map_or(false, |ext| ext == "so") { - // Finally, set the correct .interp for binaries - let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker"); - // FIXME: can we support utf8 here? `args` doesn't accept Vec, only OsString ... - let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path)))); - patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]); - } - - let _ = try_run(self, patchelf.arg(fname)); - } - - fn download_file(&self, url: &str, dest_path: &Path, help_on_error: &str) { - self.verbose(&format!("download {url}")); - // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/. - let tempfile = self.tempdir().join(dest_path.file_name().unwrap()); - // While bootstrap itself only supports http and https downloads, downstream forks might - // need to download components from other protocols. The match allows them adding more - // protocols without worrying about merge conflicts if we change the HTTP implementation. - match url.split_once("://").map(|(proto, _)| proto) { - Some("http") | Some("https") => { - self.download_http_with_retries(&tempfile, url, help_on_error) - } - Some(other) => panic!("unsupported protocol {other} in {url}"), - None => panic!("no protocol in {url}"), - } - t!(std::fs::rename(&tempfile, dest_path)); - } - - fn download_http_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) { - println!("downloading {url}"); - // Try curl. If that fails and we are on windows, fallback to PowerShell. - let mut curl = Command::new("curl"); - curl.args(&[ - "-y", - "30", - "-Y", - "10", // timeout if speed is < 10 bytes/sec for > 30 seconds - "--connect-timeout", - "30", // timeout if cannot connect within 30 seconds - "-o", - tempfile.to_str().unwrap(), - "--retry", - "3", - "-SRf", - ]); - // Don't print progress in CI; the \r wrapping looks bad and downloads don't take long enough for progress to be useful. - if CiEnv::is_ci() { - curl.arg("-s"); - } else { - curl.arg("--progress-bar"); - } - curl.arg(url); - if !self.check_run(&mut curl) { - if self.build.contains("windows-msvc") { - eprintln!("Fallback to PowerShell"); - for _ in 0..3 { - if try_run(self, Command::new("PowerShell.exe").args(&[ - "/nologo", - "-Command", - "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", - &format!( - "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", - url, tempfile.to_str().expect("invalid UTF-8 not supported with powershell downloads"), - ), - ])).is_err() { - return; - } - eprintln!("\nspurious failure, trying again"); - } - } - if !help_on_error.is_empty() { - eprintln!("{help_on_error}"); - } - crate::exit!(1); - } - } - - fn unpack(&self, tarball: &Path, dst: &Path, pattern: &str) { - eprintln!("extracting {} to {}", tarball.display(), dst.display()); - if !dst.exists() { - t!(fs::create_dir_all(dst)); - } - - // `tarball` ends with `.tar.xz`; strip that suffix - // example: `rust-dev-nightly-x86_64-unknown-linux-gnu` - let uncompressed_filename = - Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap(); - let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap()); - - // decompress the file - let data = t!(File::open(tarball), format!("file {} not found", tarball.display())); - let decompressor = XzDecoder::new(BufReader::new(data)); - - let mut tar = tar::Archive::new(decompressor); - - // `compile::Sysroot` needs to know the contents of the `rustc-dev` tarball to avoid adding - // it to the sysroot unless it was explicitly requested. But parsing the 100 MB tarball is slow. - // Cache the entries when we extract it so we only have to read it once. - let mut recorded_entries = - if dst.ends_with("ci-rustc") { recorded_entries(dst, pattern) } else { None }; - - for member in t!(tar.entries()) { - let mut member = t!(member); - let original_path = t!(member.path()).into_owned(); - // skip the top-level directory - if original_path == directory_prefix { - continue; - } - let mut short_path = t!(original_path.strip_prefix(directory_prefix)); - if !short_path.starts_with(pattern) { - continue; - } - short_path = t!(short_path.strip_prefix(pattern)); - let dst_path = dst.join(short_path); - self.verbose(&format!("extracting {} to {}", original_path.display(), dst.display())); - if !t!(member.unpack_in(dst)) { - panic!("path traversal attack ??"); - } - if let Some(record) = &mut recorded_entries { - t!(writeln!(record, "{}", short_path.to_str().unwrap())); - } - let src_path = dst.join(original_path); - if src_path.is_dir() && dst_path.exists() { - continue; - } - t!(fs::rename(src_path, dst_path)); - } - let dst_dir = dst.join(directory_prefix); - if dst_dir.exists() { - t!(fs::remove_dir_all(&dst_dir), format!("failed to remove {}", dst_dir.display())); - } - } - - /// Returns whether the SHA256 checksum of `path` matches `expected`. - fn verify(&self, path: &Path, expected: &str) -> bool { - use sha2::Digest; - - self.verbose(&format!("verifying {}", path.display())); - let mut hasher = sha2::Sha256::new(); - // FIXME: this is ok for rustfmt (4.1 MB large at time of writing), but it seems memory-intensive for rustc and larger components. - // Consider using streaming IO instead? - let contents = if self.dry_run() { vec![] } else { t!(fs::read(path)) }; - hasher.update(&contents); - let found = hex::encode(hasher.finalize().as_slice()); - let verified = found == expected; - if !verified && !self.dry_run() { - println!( - "invalid checksum: \n\ - found: {found}\n\ - expected: {expected}", - ); - } - return verified; - } -} - -fn recorded_entries(dst: &Path, pattern: &str) -> Option> { - let name = if pattern == "rustc-dev" { - ".rustc-dev-contents" - } else if pattern.starts_with("rust-std") { - ".rust-std-contents" - } else { - return None; - }; - Some(BufWriter::new(t!(File::create(dst.join(name))))) -} - -enum DownloadSource { - CI, - Dist, -} - -/// Functions that are only ever called once, but named for clarify and to avoid thousand-line functions. -impl Config { - pub(crate) fn maybe_download_rustfmt(&self) -> Option { - let RustfmtMetadata { date, version } = self.stage0_metadata.rustfmt.as_ref()?; - let channel = format!("{version}-{date}"); - - let host = self.build; - let bin_root = self.out.join(host.triple).join("rustfmt"); - let rustfmt_path = bin_root.join("bin").join(exe("rustfmt", host)); - let rustfmt_stamp = bin_root.join(".rustfmt-stamp"); - if rustfmt_path.exists() && !program_out_of_date(&rustfmt_stamp, &channel) { - return Some(rustfmt_path); - } - - self.download_component( - DownloadSource::Dist, - format!("rustfmt-{version}-{build}.tar.xz", build = host.triple), - "rustfmt-preview", - &date, - "rustfmt", - ); - self.download_component( - DownloadSource::Dist, - format!("rustc-{version}-{build}.tar.xz", build = host.triple), - "rustc", - &date, - "rustfmt", - ); - - if self.should_fix_bins_and_dylibs() { - self.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt")); - self.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt")); - let lib_dir = bin_root.join("lib"); - for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) { - let lib = t!(lib); - if lib.path().extension() == Some(OsStr::new("so")) { - self.fix_bin_or_dylib(&lib.path()); - } - } - } - - self.create(&rustfmt_stamp, &channel); - Some(rustfmt_path) - } - - pub(crate) fn ci_rust_std_contents(&self) -> Vec { - self.ci_component_contents(".rust-std-contents") - } - - pub(crate) fn ci_rustc_dev_contents(&self) -> Vec { - self.ci_component_contents(".rustc-dev-contents") - } - - fn ci_component_contents(&self, stamp_file: &str) -> Vec { - assert!(self.download_rustc()); - if self.dry_run() { - return vec![]; - } - - let ci_rustc_dir = self.ci_rustc_dir(); - let stamp_file = ci_rustc_dir.join(stamp_file); - let contents_file = t!(File::open(&stamp_file), stamp_file.display().to_string()); - t!(BufReader::new(contents_file).lines().collect()) - } - - pub(crate) fn download_ci_rustc(&self, commit: &str) { - self.verbose(&format!("using downloaded stage2 artifacts from CI (commit {commit})")); - - let version = self.artifact_version_part(commit); - // download-rustc doesn't need its own cargo, it can just use beta's. But it does need the - // `rustc_private` crates for tools. - let extra_components = ["rustc-dev"]; - - self.download_toolchain( - &version, - "ci-rustc", - &format!("{commit}-{}", self.llvm_assertions), - &extra_components, - Self::download_ci_component, - ); - } - - pub(crate) fn download_beta_toolchain(&self) { - self.verbose("downloading stage0 beta artifacts"); - - let date = &self.stage0_metadata.compiler.date; - let version = &self.stage0_metadata.compiler.version; - let extra_components = ["cargo"]; - - let download_beta_component = |config: &Config, filename, prefix: &_, date: &_| { - config.download_component(DownloadSource::Dist, filename, prefix, date, "stage0") - }; - - self.download_toolchain( - version, - "stage0", - date, - &extra_components, - download_beta_component, - ); - } - - fn download_toolchain( - &self, - version: &str, - sysroot: &str, - stamp_key: &str, - extra_components: &[&str], - download_component: fn(&Config, String, &str, &str), - ) { - let host = self.build.triple; - let bin_root = self.out.join(host).join(sysroot); - let rustc_stamp = bin_root.join(".rustc-stamp"); - - if !bin_root.join("bin").join(exe("rustc", self.build)).exists() - || program_out_of_date(&rustc_stamp, stamp_key) - { - if bin_root.exists() { - t!(fs::remove_dir_all(&bin_root)); - } - let filename = format!("rust-std-{version}-{host}.tar.xz"); - let pattern = format!("rust-std-{host}"); - download_component(self, filename, &pattern, stamp_key); - let filename = format!("rustc-{version}-{host}.tar.xz"); - download_component(self, filename, "rustc", stamp_key); - - for component in extra_components { - let filename = format!("{component}-{version}-{host}.tar.xz"); - download_component(self, filename, component, stamp_key); - } - - if self.should_fix_bins_and_dylibs() { - self.fix_bin_or_dylib(&bin_root.join("bin").join("rustc")); - self.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc")); - self.fix_bin_or_dylib( - &bin_root.join("libexec").join("rust-analyzer-proc-macro-srv"), - ); - let lib_dir = bin_root.join("lib"); - for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) { - let lib = t!(lib); - if lib.path().extension() == Some(OsStr::new("so")) { - self.fix_bin_or_dylib(&lib.path()); - } - } - } - - t!(fs::write(rustc_stamp, stamp_key)); - } - } - - /// Download a single component of a CI-built toolchain (not necessarily a published nightly). - // NOTE: intentionally takes an owned string to avoid downloading multiple times by accident - fn download_ci_component(&self, filename: String, prefix: &str, commit_with_assertions: &str) { - Self::download_component( - self, - DownloadSource::CI, - filename, - prefix, - commit_with_assertions, - "ci-rustc", - ) - } - - fn download_component( - &self, - mode: DownloadSource, - filename: String, - prefix: &str, - key: &str, - destination: &str, - ) { - let cache_dst = self.out.join("cache"); - let cache_dir = cache_dst.join(key); - if !cache_dir.exists() { - t!(fs::create_dir_all(&cache_dir)); - } - - let bin_root = self.out.join(self.build.triple).join(destination); - let tarball = cache_dir.join(&filename); - let (base_url, url, should_verify) = match mode { - DownloadSource::CI => { - let dist_server = if self.llvm_assertions { - self.stage0_metadata.config.artifacts_with_llvm_assertions_server.clone() - } else { - self.stage0_metadata.config.artifacts_server.clone() - }; - let url = format!( - "{}/{filename}", - key.strip_suffix(&format!("-{}", self.llvm_assertions)).unwrap() - ); - (dist_server, url, false) - } - DownloadSource::Dist => { - let dist_server = env::var("RUSTUP_DIST_SERVER") - .unwrap_or(self.stage0_metadata.config.dist_server.to_string()); - // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json - (dist_server, format!("dist/{key}/{filename}"), true) - } - }; - - // For the beta compiler, put special effort into ensuring the checksums are valid. - // FIXME: maybe we should do this for download-rustc as well? but it would be a pain to update - // this on each and every nightly ... - let checksum = if should_verify { - let error = format!( - "src/stage0.json doesn't contain a checksum for {url}. \ - Pre-built artifacts might not be available for this \ - target at this time, see https://doc.rust-lang.org/nightly\ - /rustc/platform-support.html for more information." - ); - let sha256 = self.stage0_metadata.checksums_sha256.get(&url).expect(&error); - if tarball.exists() { - if self.verify(&tarball, sha256) { - self.unpack(&tarball, &bin_root, prefix); - return; - } else { - self.verbose(&format!( - "ignoring cached file {} due to failed verification", - tarball.display() - )); - self.remove(&tarball); - } - } - Some(sha256) - } else if tarball.exists() { - self.unpack(&tarball, &bin_root, prefix); - return; - } else { - None - }; - - let mut help_on_error = ""; - if destination == "ci-rustc" { - help_on_error = "error: failed to download pre-built rustc from CI - -note: old builds get deleted after a certain time -help: if trying to compile an old commit of rustc, disable `download-rustc` in config.toml: - -[rust] -download-rustc = false -"; - } - self.download_file(&format!("{base_url}/{url}"), &tarball, help_on_error); - if let Some(sha256) = checksum { - if !self.verify(&tarball, sha256) { - panic!("failed to verify {}", tarball.display()); - } - } - - self.unpack(&tarball, &bin_root, prefix); - } - - pub(crate) fn maybe_download_ci_llvm(&self) { - if !self.llvm_from_ci { - return; - } - let llvm_root = self.ci_llvm_root(); - let llvm_stamp = llvm_root.join(".llvm-stamp"); - let llvm_sha = detect_llvm_sha(&self, self.rust_info.is_managed_git_subrepository()); - let key = format!("{}{}", llvm_sha, self.llvm_assertions); - if program_out_of_date(&llvm_stamp, &key) && !self.dry_run() { - self.download_ci_llvm(&llvm_sha); - if self.should_fix_bins_and_dylibs() { - for entry in t!(fs::read_dir(llvm_root.join("bin"))) { - self.fix_bin_or_dylib(&t!(entry).path()); - } - } - - // Update the timestamp of llvm-config to force rustc_llvm to be - // rebuilt. This is a hacky workaround for a deficiency in Cargo where - // the rerun-if-changed directive doesn't handle changes very well. - // https://github.com/rust-lang/cargo/issues/10791 - // Cargo only compares the timestamp of the file relative to the last - // time `rustc_llvm` build script ran. However, the timestamps of the - // files in the tarball are in the past, so it doesn't trigger a - // rebuild. - let now = filetime::FileTime::from_system_time(std::time::SystemTime::now()); - let llvm_config = llvm_root.join("bin").join(exe("llvm-config", self.build)); - t!(filetime::set_file_times(&llvm_config, now, now)); - - if self.should_fix_bins_and_dylibs() { - let llvm_lib = llvm_root.join("lib"); - for entry in t!(fs::read_dir(&llvm_lib)) { - let lib = t!(entry).path(); - if lib.extension().map_or(false, |ext| ext == "so") { - self.fix_bin_or_dylib(&lib); - } - } - } - - t!(fs::write(llvm_stamp, key)); - } - } - - fn download_ci_llvm(&self, llvm_sha: &str) { - let llvm_assertions = self.llvm_assertions; - - let cache_prefix = format!("llvm-{llvm_sha}-{llvm_assertions}"); - let cache_dst = self.out.join("cache"); - let rustc_cache = cache_dst.join(cache_prefix); - if !rustc_cache.exists() { - t!(fs::create_dir_all(&rustc_cache)); - } - let base = if llvm_assertions { - &self.stage0_metadata.config.artifacts_with_llvm_assertions_server - } else { - &self.stage0_metadata.config.artifacts_server - }; - let version = self.artifact_version_part(llvm_sha); - let filename = format!("rust-dev-{}-{}.tar.xz", version, self.build.triple); - let tarball = rustc_cache.join(&filename); - if !tarball.exists() { - let help_on_error = "error: failed to download llvm from ci - - help: old builds get deleted after a certain time - help: if trying to compile an old commit of rustc, disable `download-ci-llvm` in config.toml: - - [llvm] - download-ci-llvm = false - "; - self.download_file(&format!("{base}/{llvm_sha}/{filename}"), &tarball, help_on_error); - } - let llvm_root = self.ci_llvm_root(); - self.unpack(&tarball, &llvm_root, "rust-dev"); - } -} diff --git a/src/bootstrap/dylib_util.rs b/src/bootstrap/dylib_util.rs deleted file mode 100644 index b14c0bed6..000000000 --- a/src/bootstrap/dylib_util.rs +++ /dev/null @@ -1,30 +0,0 @@ -// Various utilities for working with dylib paths. -// -// This file is meant to be included directly to avoid a dependency on the bootstrap library from -// the rustc and rustdoc wrappers. This improves compilation time by reducing the linking time. - -/// Returns the environment variable which the dynamic library lookup path -/// resides in for this platform. -pub fn dylib_path_var() -> &'static str { - if cfg!(target_os = "windows") { - "PATH" - } else if cfg!(target_os = "macos") { - "DYLD_LIBRARY_PATH" - } else if cfg!(target_os = "haiku") { - "LIBRARY_PATH" - } else if cfg!(target_os = "aix") { - "LIBPATH" - } else { - "LD_LIBRARY_PATH" - } -} - -/// Parses the `dylib_path_var()` environment variable, returning a list of -/// paths that are members of this lookup path. -pub fn dylib_path() -> Vec { - let var = match env::var_os(dylib_path_var()) { - Some(v) => v, - None => return vec![], - }; - env::split_paths(&var).collect() -} diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs deleted file mode 100644 index e0291e407..000000000 --- a/src/bootstrap/flags.rs +++ /dev/null @@ -1,565 +0,0 @@ -//! Command-line interface of the rustbuild build system. -//! -//! This module implements the command-line parsing of the build system which -//! has various flags to configure how it's run. - -use std::path::{Path, PathBuf}; - -use clap::{CommandFactory, Parser, ValueEnum}; - -use crate::builder::{Builder, Kind}; -use crate::config::{target_selection_list, Config, TargetSelectionList}; -use crate::setup::Profile; -use crate::{Build, DocTests}; - -#[derive(Copy, Clone, Default, Debug, ValueEnum)] -pub enum Color { - Always, - Never, - #[default] - Auto, -} - -/// Whether to deny warnings, emit them as warnings, or use the default behavior -#[derive(Copy, Clone, Default, Debug, ValueEnum)] -pub enum Warnings { - Deny, - Warn, - #[default] - Default, -} - -/// Deserialized version of all flags for this compile. -#[derive(Debug, Parser)] -#[clap( - override_usage = "x.py [options] [...]", - disable_help_subcommand(true), - about = "", - next_line_help(false) -)] -pub struct Flags { - #[command(subcommand)] - pub cmd: Subcommand, - - #[arg(global(true), short, long, action = clap::ArgAction::Count)] - /// use verbose output (-vv for very verbose) - pub verbose: u8, // each extra -v after the first is passed to Cargo - #[arg(global(true), short, long)] - /// use incremental compilation - pub incremental: bool, - #[arg(global(true), long, value_hint = clap::ValueHint::FilePath, value_name = "FILE")] - /// TOML configuration file for build - pub config: Option, - #[arg(global(true), long, value_hint = clap::ValueHint::DirPath, value_name = "DIR")] - /// Build directory, overrides `build.build-dir` in `config.toml` - pub build_dir: Option, - - #[arg(global(true), long, value_hint = clap::ValueHint::Other, value_name = "BUILD")] - /// build target of the stage0 compiler - pub build: Option, - - #[arg(global(true), long, value_hint = clap::ValueHint::Other, value_name = "HOST", value_parser = target_selection_list)] - /// host targets to build - pub host: Option, - - #[arg(global(true), long, value_hint = clap::ValueHint::Other, value_name = "TARGET", value_parser = target_selection_list)] - /// target targets to build - pub target: Option, - - #[arg(global(true), long, value_name = "PATH")] - /// build paths to exclude - pub exclude: Vec, // keeping for client backward compatibility - #[arg(global(true), long, value_name = "PATH")] - /// build paths to skip - pub skip: Vec, - #[arg(global(true), long)] - /// include default paths in addition to the provided ones - pub include_default_paths: bool, - - #[arg(global(true), value_hint = clap::ValueHint::Other, long)] - pub rustc_error_format: Option, - - #[arg(global(true), long, value_hint = clap::ValueHint::CommandString, value_name = "CMD")] - /// command to run on failure - pub on_fail: Option, - #[arg(global(true), long)] - /// dry run; don't build anything - pub dry_run: bool, - #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "N")] - /// stage to build (indicates compiler to use/test, e.g., stage 0 uses the - /// bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.) - pub stage: Option, - - #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "N")] - /// stage(s) to keep without recompiling - /// (pass multiple times to keep e.g., both stages 0 and 1) - pub keep_stage: Vec, - #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "N")] - /// stage(s) of the standard library to keep without recompiling - /// (pass multiple times to keep e.g., both stages 0 and 1) - pub keep_stage_std: Vec, - #[arg(global(true), long, value_hint = clap::ValueHint::DirPath, value_name = "DIR")] - /// path to the root of the rust checkout - pub src: Option, - - #[arg( - global(true), - short, - long, - value_hint = clap::ValueHint::Other, - default_value_t = std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get), - value_name = "JOBS" - )] - /// number of jobs to run in parallel - pub jobs: usize, - // This overrides the deny-warnings configuration option, - // which passes -Dwarnings to the compiler invocations. - #[arg(global(true), long)] - #[clap(value_enum, default_value_t=Warnings::Default, value_name = "deny|warn")] - /// if value is deny, will deny warnings - /// if value is warn, will emit warnings - /// otherwise, use the default configured behaviour - pub warnings: Warnings, - - #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "FORMAT")] - /// rustc error format - pub error_format: Option, - #[arg(global(true), long)] - /// use message-format=json - pub json_output: bool, - - #[arg(global(true), long, value_name = "STYLE")] - #[clap(value_enum, default_value_t = Color::Auto)] - /// whether to use color in cargo and rustc output - pub color: Color, - - /// whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml - #[arg(global(true), long, value_name = "VALUE")] - pub llvm_skip_rebuild: Option, - /// generate PGO profile with rustc build - #[arg(global(true), value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] - pub rust_profile_generate: Option, - /// use PGO profile for rustc build - #[arg(global(true), value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] - pub rust_profile_use: Option, - /// use PGO profile for LLVM build - #[arg(global(true), value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] - pub llvm_profile_use: Option, - // LLVM doesn't support a custom location for generating profile - // information. - // - // llvm_out/build/profiles/ is the location this writes to. - /// generate PGO profile with llvm built for rustc - #[arg(global(true), long)] - pub llvm_profile_generate: bool, - /// Additional reproducible artifacts that should be added to the reproducible artifacts archive. - #[arg(global(true), long)] - pub reproducible_artifact: Vec, - #[arg(global(true))] - /// paths for the subcommand - pub paths: Vec, - /// override options in config.toml - #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "section.option=value")] - pub set: Vec, - /// arguments passed to subcommands - #[arg(global(true), last(true), value_name = "ARGS")] - pub free_args: Vec, -} - -impl Flags { - pub fn parse(args: &[String]) -> Self { - let first = String::from("x.py"); - let it = std::iter::once(&first).chain(args.iter()); - // We need to check for ` -h -v`, in which case we list the paths - #[derive(Parser)] - #[clap(disable_help_flag(true))] - struct HelpVerboseOnly { - #[arg(short, long)] - help: bool, - #[arg(global(true), short, long, action = clap::ArgAction::Count)] - pub verbose: u8, - #[arg(value_enum)] - cmd: Kind, - } - if let Ok(HelpVerboseOnly { help: true, verbose: 1.., cmd: subcommand }) = - HelpVerboseOnly::try_parse_from(it.clone()) - { - println!("note: updating submodules before printing available paths"); - let config = Config::parse(&[String::from("build")]); - let build = Build::new(config); - let paths = Builder::get_help(&build, subcommand); - if let Some(s) = paths { - println!("{s}"); - } else { - panic!("No paths available for subcommand `{}`", subcommand.as_str()); - } - crate::exit!(0); - } - - Flags::parse_from(it) - } -} - -#[derive(Debug, Clone, Default, clap::Subcommand)] -pub enum Subcommand { - #[clap(aliases = ["b"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories to the crates - and/or artifacts to compile. For example, for a quick build of a usable - compiler: - ./x.py build --stage 1 library/std - This will build a compiler and standard library from the local source code. - Once this is done, build/$ARCH/stage1 contains a usable compiler. - If no arguments are passed then the default artifacts for that stage are - compiled. For example: - ./x.py build --stage 0 - ./x.py build ")] - /// Compile either the compiler or libraries - #[default] - Build, - #[clap(aliases = ["c"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories to the crates - and/or artifacts to compile. For example: - ./x.py check library/std - If no arguments are passed then many artifacts are checked.")] - /// Compile either the compiler or libraries, using cargo check - Check { - #[arg(long)] - /// Check all targets - all_targets: bool, - }, - /// Run Clippy (uses rustup/cargo-installed clippy binary) - #[clap(long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories to the crates - and/or artifacts to run clippy against. For example: - ./x.py clippy library/core - ./x.py clippy library/core library/proc_macro")] - Clippy { - #[arg(long)] - fix: bool, - /// clippy lints to allow - #[arg(global(true), short = 'A', action = clap::ArgAction::Append, value_name = "LINT")] - allow: Vec, - /// clippy lints to deny - #[arg(global(true), short = 'D', action = clap::ArgAction::Append, value_name = "LINT")] - deny: Vec, - /// clippy lints to warn on - #[arg(global(true), short = 'W', action = clap::ArgAction::Append, value_name = "LINT")] - warn: Vec, - /// clippy lints to forbid - #[arg(global(true), short = 'F', action = clap::ArgAction::Append, value_name = "LINT")] - forbid: Vec, - }, - /// Run cargo fix - #[clap(long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories to the crates - and/or artifacts to run `cargo fix` against. For example: - ./x.py fix library/core - ./x.py fix library/core library/proc_macro")] - Fix, - #[clap( - name = "fmt", - long_about = "\n - Arguments: - This subcommand optionally accepts a `--check` flag which succeeds if formatting is correct and - fails if it is not. For example: - ./x.py fmt - ./x.py fmt --check" - )] - /// Run rustfmt - Format { - /// check formatting instead of applying - #[arg(long)] - check: bool, - }, - #[clap(aliases = ["d"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories of documentation - to build. For example: - ./x.py doc src/doc/book - ./x.py doc src/doc/nomicon - ./x.py doc src/doc/book library/std - ./x.py doc library/std --json - ./x.py doc library/std --open - If no arguments are passed then everything is documented: - ./x.py doc - ./x.py doc --stage 1")] - /// Build documentation - Doc { - #[arg(long)] - /// open the docs in a browser - open: bool, - #[arg(long)] - /// render the documentation in JSON format in addition to the usual HTML format - json: bool, - }, - #[clap(aliases = ["t"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to test directories that - should be compiled and run. For example: - ./x.py test tests/ui - ./x.py test library/std --test-args hash_map - ./x.py test library/std --stage 0 --no-doc - ./x.py test tests/ui --bless - ./x.py test tests/ui --compare-mode next-solver - Note that `test tests/* --stage N` does NOT depend on `build compiler/rustc --stage N`; - just like `build library/std --stage N` it tests the compiler produced by the previous - stage. - Execute tool tests with a tool name argument: - ./x.py test tidy - If no arguments are passed then the complete artifacts for that stage are - compiled and tested. - ./x.py test - ./x.py test --stage 1")] - /// Build and run some test suites - Test { - #[arg(long)] - /// run all tests regardless of failure - no_fail_fast: bool, - #[arg(long, value_name = "SUBSTRING")] - /// skips tests matching SUBSTRING, if supported by test tool. May be passed multiple times - skip: Vec, - #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] - /// extra arguments to be passed for the test tool being used - /// (e.g. libtest, compiletest or rustdoc) - test_args: Vec, - /// extra options to pass the compiler when running tests - #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] - rustc_args: Vec, - #[arg(long)] - /// do not run doc tests - no_doc: bool, - #[arg(long)] - /// only run doc tests - doc: bool, - #[arg(long)] - /// whether to automatically update stderr/stdout files - bless: bool, - #[arg(long)] - /// comma-separated list of other files types to check (accepts py, py:lint, - /// py:fmt, shell) - extra_checks: Option, - #[arg(long)] - /// rerun tests even if the inputs are unchanged - force_rerun: bool, - #[arg(long)] - /// only run tests that result has been changed - only_modified: bool, - #[arg(long, value_name = "COMPARE MODE")] - /// mode describing what file the actual ui output will be compared to - compare_mode: Option, - #[arg(long, value_name = "check | build | run")] - /// force {check,build,run}-pass tests to this mode. - pass: Option, - #[arg(long, value_name = "auto | always | never")] - /// whether to execute run-* tests - run: Option, - #[arg(long)] - /// enable this to generate a Rustfix coverage file, which is saved in - /// `//rustfix_missing_coverage.txt` - rustfix_coverage: bool, - }, - /// Build and run some benchmarks - Bench { - #[arg(long, allow_hyphen_values(true))] - test_args: Vec, - }, - /// Clean out build directories - Clean { - #[arg(long)] - /// Clean the entire build directory (not used by default) - all: bool, - #[arg(long, value_name = "N")] - /// Clean a specific stage without touching other artifacts. By default, every stage is cleaned if this option is not used. - stage: Option, - }, - /// Build distribution artifacts - Dist, - /// Install distribution artifacts - Install, - #[clap(aliases = ["r"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to tools to build and run. For - example: - ./x.py run src/tools/expand-yaml-anchors - At least a tool needs to be called.")] - /// Run tools contained in this repository - Run { - /// arguments for the tool - #[arg(long, allow_hyphen_values(true))] - args: Vec, - }, - /// Set up the environment for development - #[clap(long_about = format!( - "\n -x.py setup creates a `config.toml` which changes the defaults for x.py itself, -as well as setting up a git pre-push hook, VS Code config and toolchain link. -Arguments: - This subcommand accepts a 'profile' to use for builds. For example: - ./x.py setup library - The profile is optional and you will be prompted interactively if it is not given. - The following profiles are available: -{} - To only set up the git hook, VS Code config or toolchain link, you may use - ./x.py setup hook - ./x.py setup vscode - ./x.py setup link", Profile::all_for_help(" ").trim_end()))] - Setup { - /// Either the profile for `config.toml` or another setup action. - /// May be omitted to set up interactively - #[arg(value_name = "|hook|vscode|link")] - profile: Option, - }, - /// Suggest a subset of tests to run, based on modified files - #[clap(long_about = "\n")] - Suggest { - /// run suggested tests - #[arg(long)] - run: bool, - }, -} - -impl Subcommand { - pub fn kind(&self) -> Kind { - match self { - Subcommand::Bench { .. } => Kind::Bench, - Subcommand::Build { .. } => Kind::Build, - Subcommand::Check { .. } => Kind::Check, - Subcommand::Clippy { .. } => Kind::Clippy, - Subcommand::Doc { .. } => Kind::Doc, - Subcommand::Fix { .. } => Kind::Fix, - Subcommand::Format { .. } => Kind::Format, - Subcommand::Test { .. } => Kind::Test, - Subcommand::Clean { .. } => Kind::Clean, - Subcommand::Dist { .. } => Kind::Dist, - Subcommand::Install { .. } => Kind::Install, - Subcommand::Run { .. } => Kind::Run, - Subcommand::Setup { .. } => Kind::Setup, - Subcommand::Suggest { .. } => Kind::Suggest, - } - } - - pub fn rustc_args(&self) -> Vec<&str> { - match *self { - Subcommand::Test { ref rustc_args, .. } => { - rustc_args.iter().flat_map(|s| s.split_whitespace()).collect() - } - _ => vec![], - } - } - - pub fn fail_fast(&self) -> bool { - match *self { - Subcommand::Test { no_fail_fast, .. } => !no_fail_fast, - _ => false, - } - } - - pub fn doc_tests(&self) -> DocTests { - match *self { - Subcommand::Test { doc, no_doc, .. } => { - if doc { - DocTests::Only - } else if no_doc { - DocTests::No - } else { - DocTests::Yes - } - } - _ => DocTests::Yes, - } - } - - pub fn bless(&self) -> bool { - match *self { - Subcommand::Test { bless, .. } => bless, - _ => false, - } - } - - pub fn extra_checks(&self) -> Option<&str> { - match *self { - Subcommand::Test { ref extra_checks, .. } => extra_checks.as_ref().map(String::as_str), - _ => None, - } - } - - pub fn only_modified(&self) -> bool { - match *self { - Subcommand::Test { only_modified, .. } => only_modified, - _ => false, - } - } - - pub fn force_rerun(&self) -> bool { - match *self { - Subcommand::Test { force_rerun, .. } => force_rerun, - _ => false, - } - } - - pub fn rustfix_coverage(&self) -> bool { - match *self { - Subcommand::Test { rustfix_coverage, .. } => rustfix_coverage, - _ => false, - } - } - - pub fn compare_mode(&self) -> Option<&str> { - match *self { - Subcommand::Test { ref compare_mode, .. } => compare_mode.as_ref().map(|s| &s[..]), - _ => None, - } - } - - pub fn pass(&self) -> Option<&str> { - match *self { - Subcommand::Test { ref pass, .. } => pass.as_ref().map(|s| &s[..]), - _ => None, - } - } - - pub fn run(&self) -> Option<&str> { - match *self { - Subcommand::Test { ref run, .. } => run.as_ref().map(|s| &s[..]), - _ => None, - } - } - - pub fn open(&self) -> bool { - match *self { - Subcommand::Doc { open, .. } => open, - _ => false, - } - } - - pub fn json(&self) -> bool { - match *self { - Subcommand::Doc { json, .. } => json, - _ => false, - } - } -} - -/// Returns the shell completion for a given shell, if the result differs from the current -/// content of `path`. If `path` does not exist, always returns `Some`. -pub fn get_completion(shell: G, path: &Path) -> Option { - let mut cmd = Flags::command(); - let current = if !path.exists() { - String::new() - } else { - std::fs::read_to_string(path).unwrap_or_else(|_| { - eprintln!("couldn't read {}", path.display()); - crate::exit!(1) - }) - }; - let mut buf = Vec::new(); - clap_complete::generate(shell, &mut cmd, "x.py", &mut buf); - if buf == current.as_bytes() { - return None; - } - Some(String::from_utf8(buf).expect("completion script should be UTF-8")) -} diff --git a/src/bootstrap/format.rs b/src/bootstrap/format.rs deleted file mode 100644 index 11f2762f7..000000000 --- a/src/bootstrap/format.rs +++ /dev/null @@ -1,322 +0,0 @@ -//! Runs rustfmt on the repository. - -use crate::builder::Builder; -use crate::util::{output, program_out_of_date, t}; -use build_helper::ci::CiEnv; -use build_helper::git::get_git_modified_files; -use ignore::WalkBuilder; -use std::collections::VecDeque; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use std::sync::mpsc::SyncSender; - -fn rustfmt(src: &Path, rustfmt: &Path, paths: &[PathBuf], check: bool) -> impl FnMut(bool) -> bool { - let mut cmd = Command::new(&rustfmt); - // avoid the submodule config paths from coming into play, - // we only allow a single global config for the workspace for now - cmd.arg("--config-path").arg(&src.canonicalize().unwrap()); - cmd.arg("--edition").arg("2021"); - cmd.arg("--unstable-features"); - cmd.arg("--skip-children"); - if check { - cmd.arg("--check"); - } - cmd.args(paths); - let cmd_debug = format!("{cmd:?}"); - let mut cmd = cmd.spawn().expect("running rustfmt"); - // poor man's async: return a closure that'll wait for rustfmt's completion - move |block: bool| -> bool { - if !block { - match cmd.try_wait() { - Ok(Some(_)) => {} - _ => return false, - } - } - let status = cmd.wait().unwrap(); - if !status.success() { - eprintln!( - "Running `{}` failed.\nIf you're running `tidy`, \ - try again with `--bless`. Or, if you just want to format \ - code, run `./x.py fmt` instead.", - cmd_debug, - ); - crate::exit!(1); - } - true - } -} - -fn get_rustfmt_version(build: &Builder<'_>) -> Option<(String, PathBuf)> { - let stamp_file = build.out.join("rustfmt.stamp"); - - let mut cmd = Command::new(match build.initial_rustfmt() { - Some(p) => p, - None => return None, - }); - cmd.arg("--version"); - let output = match cmd.output() { - Ok(status) => status, - Err(_) => return None, - }; - if !output.status.success() { - return None; - } - Some((String::from_utf8(output.stdout).unwrap(), stamp_file)) -} - -/// Return whether the format cache can be reused. -fn verify_rustfmt_version(build: &Builder<'_>) -> bool { - let Some((version, stamp_file)) = get_rustfmt_version(build) else { - return false; - }; - !program_out_of_date(&stamp_file, &version) -} - -/// Updates the last rustfmt version used -fn update_rustfmt_version(build: &Builder<'_>) { - let Some((version, stamp_file)) = get_rustfmt_version(build) else { - return; - }; - t!(std::fs::write(stamp_file, version)) -} - -/// Returns the Rust files modified between the `merge-base` of HEAD and -/// rust-lang/master and what is now on the disk. -/// -/// Returns `None` if all files should be formatted. -fn get_modified_rs_files(build: &Builder<'_>) -> Result>, String> { - if !verify_rustfmt_version(build) { - return Ok(None); - } - - get_git_modified_files(Some(&build.config.src), &vec!["rs"]) -} - -#[derive(serde_derive::Deserialize)] -struct RustfmtConfig { - ignore: Vec, -} - -pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) { - if build.config.dry_run() { - return; - } - let mut builder = ignore::types::TypesBuilder::new(); - builder.add_defaults(); - builder.select("rust"); - let matcher = builder.build().unwrap(); - let rustfmt_config = build.src.join("rustfmt.toml"); - if !rustfmt_config.exists() { - eprintln!("Not running formatting checks; rustfmt.toml does not exist."); - eprintln!("This may happen in distributed tarballs."); - return; - } - let rustfmt_config = t!(std::fs::read_to_string(&rustfmt_config)); - let rustfmt_config: RustfmtConfig = t!(toml::from_str(&rustfmt_config)); - let mut fmt_override = ignore::overrides::OverrideBuilder::new(&build.src); - for ignore in rustfmt_config.ignore { - fmt_override.add(&format!("!{ignore}")).expect(&ignore); - } - let git_available = match Command::new("git") - .arg("--version") - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - { - Ok(status) => status.success(), - Err(_) => false, - }; - - if git_available { - let in_working_tree = match build - .config - .git() - .arg("rev-parse") - .arg("--is-inside-work-tree") - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - { - Ok(status) => status.success(), - Err(_) => false, - }; - if in_working_tree { - let untracked_paths_output = output( - build.config.git().arg("status").arg("--porcelain").arg("--untracked-files=normal"), - ); - let untracked_paths = untracked_paths_output - .lines() - .filter(|entry| entry.starts_with("??")) - .map(|entry| { - entry.split(' ').nth(1).expect("every git status entry should list a path") - }); - let mut untracked_count = 0; - for untracked_path in untracked_paths { - println!("skip untracked path {untracked_path} during rustfmt invocations"); - // The leading `/` makes it an exact match against the - // repository root, rather than a glob. Without that, if you - // have `foo.rs` in the repository root it will also match - // against anything like `compiler/rustc_foo/src/foo.rs`, - // preventing the latter from being formatted. - untracked_count += 1; - fmt_override.add(&format!("!/{untracked_path}")).expect(&untracked_path); - } - // Only check modified files locally to speed up runtime. - // We still check all files in CI to avoid bugs in `get_modified_rs_files` letting regressions slip through; - // we also care about CI time less since this is still very fast compared to building the compiler. - if !CiEnv::is_ci() && paths.is_empty() { - match get_modified_rs_files(build) { - Ok(Some(files)) => { - if files.len() <= 10 { - for file in &files { - println!("formatting modified file {file}"); - } - } else { - let pluralized = |count| if count > 1 { "files" } else { "file" }; - let untracked_msg = if untracked_count == 0 { - "".to_string() - } else { - format!( - ", skipped {} untracked {}", - untracked_count, - pluralized(untracked_count), - ) - }; - println!( - "formatting {} modified {}{}", - files.len(), - pluralized(files.len()), - untracked_msg - ); - } - for file in files { - fmt_override.add(&format!("/{file}")).expect(&file); - } - } - Ok(None) => {} - Err(err) => { - println!( - "WARN: Something went wrong when running git commands:\n{err}\n\ - Falling back to formatting all files." - ); - } - } - } - } else { - println!("Not in git tree. Skipping git-aware format checks"); - } - } else { - println!("Could not find usable git. Skipping git-aware format checks"); - } - - let fmt_override = fmt_override.build().unwrap(); - - let rustfmt_path = build.initial_rustfmt().unwrap_or_else(|| { - eprintln!("./x.py fmt is not supported on this channel"); - crate::exit!(1); - }); - assert!(rustfmt_path.exists(), "{}", rustfmt_path.display()); - let src = build.src.clone(); - let (tx, rx): (SyncSender, _) = std::sync::mpsc::sync_channel(128); - let walker = match paths.get(0) { - Some(first) => { - let find_shortcut_candidates = |p: &PathBuf| { - let mut candidates = Vec::new(); - for candidate in WalkBuilder::new(src.clone()).max_depth(Some(3)).build() { - if let Ok(entry) = candidate { - if let Some(dir_name) = p.file_name() { - if entry.path().is_dir() && entry.file_name() == dir_name { - candidates.push(entry.into_path()); - } - } - } - } - candidates - }; - - // Only try to look for shortcut candidates for single component paths like - // `std` and not for e.g. relative paths like `../library/std`. - let should_look_for_shortcut_dir = |p: &PathBuf| p.components().count() == 1; - - let mut walker = if should_look_for_shortcut_dir(first) { - if let [single_candidate] = &find_shortcut_candidates(first)[..] { - WalkBuilder::new(single_candidate) - } else { - WalkBuilder::new(first) - } - } else { - WalkBuilder::new(src.join(first)) - }; - - for path in &paths[1..] { - if should_look_for_shortcut_dir(path) { - if let [single_candidate] = &find_shortcut_candidates(path)[..] { - walker.add(single_candidate); - } else { - walker.add(path); - } - } else { - walker.add(src.join(path)); - } - } - - walker - } - None => WalkBuilder::new(src.clone()), - } - .types(matcher) - .overrides(fmt_override) - .build_parallel(); - - // there is a lot of blocking involved in spawning a child process and reading files to format. - // spawn more processes than available concurrency to keep the CPU busy - let max_processes = build.jobs() as usize * 2; - - // spawn child processes on a separate thread so we can batch entries we have received from ignore - let thread = std::thread::spawn(move || { - let mut children = VecDeque::new(); - while let Ok(path) = rx.recv() { - // try getting a few more paths from the channel to amortize the overhead of spawning processes - let paths: Vec<_> = rx.try_iter().take(7).chain(std::iter::once(path)).collect(); - - let child = rustfmt(&src, &rustfmt_path, paths.as_slice(), check); - children.push_back(child); - - // poll completion before waiting - for i in (0..children.len()).rev() { - if children[i](false) { - children.swap_remove_back(i); - break; - } - } - - if children.len() >= max_processes { - // await oldest child - children.pop_front().unwrap()(true); - } - } - - // await remaining children - for mut child in children { - child(true); - } - }); - - walker.run(|| { - let tx = tx.clone(); - Box::new(move |entry| { - let entry = t!(entry); - if entry.file_type().map_or(false, |t| t.is_file()) { - t!(tx.send(entry.into_path())); - } - ignore::WalkState::Continue - }) - }); - - drop(tx); - - thread.join().unwrap(); - if !check { - update_rustfmt_version(build); - } -} diff --git a/src/bootstrap/install.rs b/src/bootstrap/install.rs deleted file mode 100644 index 500b20b86..000000000 --- a/src/bootstrap/install.rs +++ /dev/null @@ -1,320 +0,0 @@ -//! Implementation of the install aspects of the compiler. -//! -//! This module is responsible for installing the standard library, -//! compiler, and documentation. - -use std::env; -use std::fs; -use std::path::{Component, Path, PathBuf}; -use std::process::Command; - -use crate::util::t; - -use crate::dist; -use crate::tarball::GeneratedTarball; -use crate::{Compiler, Kind}; - -use crate::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::config::{Config, TargetSelection}; - -#[cfg(target_os = "illumos")] -const SHELL: &str = "bash"; -#[cfg(not(target_os = "illumos"))] -const SHELL: &str = "sh"; - -// We have to run a few shell scripts, which choke quite a bit on both `\` -// characters and on `C:\` paths, so normalize both of them away. -fn sanitize_sh(path: &Path) -> String { - let path = path.to_str().unwrap().replace("\\", "/"); - return change_drive(unc_to_lfs(&path)).unwrap_or(path); - - fn unc_to_lfs(s: &str) -> &str { - s.strip_prefix("//?/").unwrap_or(s) - } - - fn change_drive(s: &str) -> Option { - let mut ch = s.chars(); - let drive = ch.next().unwrap_or('C'); - if ch.next() != Some(':') { - return None; - } - if ch.next() != Some('/') { - return None; - } - Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..])) - } -} - -fn is_dir_writable_for_user(dir: &PathBuf) -> bool { - let tmp = dir.join(".tmp"); - match fs::create_dir_all(&tmp) { - Ok(_) => { - fs::remove_dir_all(tmp).unwrap(); - true - } - Err(e) => { - if e.kind() == std::io::ErrorKind::PermissionDenied { - false - } else { - panic!("Failed the write access check for the current user. {}", e); - } - } - } -} - -fn install_sh( - builder: &Builder<'_>, - package: &str, - stage: u32, - host: Option, - tarball: &GeneratedTarball, -) { - let _guard = builder.msg(Kind::Install, stage, package, host, host); - - let prefix = default_path(&builder.config.prefix, "/usr/local"); - let sysconfdir = prefix.join(default_path(&builder.config.sysconfdir, "/etc")); - let destdir_env = env::var_os("DESTDIR").map(PathBuf::from); - - // Sanity checks on the write access of user. - // - // When the `DESTDIR` environment variable is present, there is no point to - // check write access for `prefix` and `sysconfdir` individually, as they - // are combined with the path from the `DESTDIR` environment variable. In - // this case, we only need to check the `DESTDIR` path, disregarding the - // `prefix` and `sysconfdir` paths. - if let Some(destdir) = &destdir_env { - assert!(is_dir_writable_for_user(destdir), "User doesn't have write access on DESTDIR."); - } else { - assert!( - is_dir_writable_for_user(&prefix), - "User doesn't have write access on `install.prefix` path in the `config.toml`.", - ); - assert!( - is_dir_writable_for_user(&sysconfdir), - "User doesn't have write access on `install.sysconfdir` path in `config.toml`." - ); - } - - let datadir = prefix.join(default_path(&builder.config.datadir, "share")); - let docdir = prefix.join(default_path(&builder.config.docdir, "share/doc/rust")); - let mandir = prefix.join(default_path(&builder.config.mandir, "share/man")); - let libdir = prefix.join(default_path(&builder.config.libdir, "lib")); - let bindir = prefix.join(&builder.config.bindir); // Default in config.rs - - let empty_dir = builder.out.join("tmp/empty_dir"); - t!(fs::create_dir_all(&empty_dir)); - - let mut cmd = Command::new(SHELL); - cmd.current_dir(&empty_dir) - .arg(sanitize_sh(&tarball.decompressed_output().join("install.sh"))) - .arg(format!("--prefix={}", prepare_dir(&destdir_env, prefix))) - .arg(format!("--sysconfdir={}", prepare_dir(&destdir_env, sysconfdir))) - .arg(format!("--datadir={}", prepare_dir(&destdir_env, datadir))) - .arg(format!("--docdir={}", prepare_dir(&destdir_env, docdir))) - .arg(format!("--bindir={}", prepare_dir(&destdir_env, bindir))) - .arg(format!("--libdir={}", prepare_dir(&destdir_env, libdir))) - .arg(format!("--mandir={}", prepare_dir(&destdir_env, mandir))) - .arg("--disable-ldconfig"); - builder.run(&mut cmd); - t!(fs::remove_dir_all(&empty_dir)); -} - -fn default_path(config: &Option, default: &str) -> PathBuf { - config.as_ref().cloned().unwrap_or_else(|| PathBuf::from(default)) -} - -fn prepare_dir(destdir_env: &Option, mut path: PathBuf) -> String { - // The DESTDIR environment variable is a standard way to install software in a subdirectory - // while keeping the original directory structure, even if the prefix or other directories - // contain absolute paths. - // - // More information on the environment variable is available here: - // https://www.gnu.org/prep/standards/html_node/DESTDIR.html - if let Some(destdir) = destdir_env { - let without_destdir = path.clone(); - path = destdir.clone(); - // Custom .join() which ignores disk roots. - for part in without_destdir.components() { - if let Component::Normal(s) = part { - path.push(s) - } - } - } - - // The installation command is not executed from the current directory, but from a temporary - // directory. To prevent relative paths from breaking this converts relative paths to absolute - // paths. std::fs::canonicalize is not used as that requires the path to actually be present. - if path.is_relative() { - path = std::env::current_dir().expect("failed to get the current directory").join(path); - assert!(path.is_absolute(), "could not make the path relative"); - } - - sanitize_sh(&path) -} - -macro_rules! install { - (($sel:ident, $builder:ident, $_config:ident), - $($name:ident, - $condition_name: ident = $path_or_alias: literal, - $default_cond:expr, - only_hosts: $only_hosts:expr, - $run_item:block $(, $c:ident)*;)+) => { - $( - #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] - pub struct $name { - pub compiler: Compiler, - pub target: TargetSelection, - } - - impl $name { - #[allow(dead_code)] - fn should_build(config: &Config) -> bool { - config.extended && config.tools.as_ref() - .map_or(true, |t| t.contains($path_or_alias)) - } - } - - impl Step for $name { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = $only_hosts; - $(const $c: bool = true;)* - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let $_config = &run.builder.config; - run.$condition_name($path_or_alias).default_condition($default_cond) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure($name { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - fn run($sel, $builder: &Builder<'_>) { - $run_item - } - })+ - } -} - -install!((self, builder, _config), - Docs, path = "src/doc", _config.docs, only_hosts: false, { - let tarball = builder.ensure(dist::Docs { host: self.target }).expect("missing docs"); - install_sh(builder, "docs", self.compiler.stage, Some(self.target), &tarball); - }; - Std, path = "library/std", true, only_hosts: false, { - for target in &builder.targets { - // `expect` should be safe, only None when host != build, but this - // only runs when host == build - let tarball = builder.ensure(dist::Std { - compiler: self.compiler, - target: *target - }).expect("missing std"); - install_sh(builder, "std", self.compiler.stage, Some(*target), &tarball); - } - }; - Cargo, alias = "cargo", Self::should_build(_config), only_hosts: true, { - let tarball = builder - .ensure(dist::Cargo { compiler: self.compiler, target: self.target }) - .expect("missing cargo"); - install_sh(builder, "cargo", self.compiler.stage, Some(self.target), &tarball); - }; - RustAnalyzer, alias = "rust-analyzer", Self::should_build(_config), only_hosts: true, { - if let Some(tarball) = - builder.ensure(dist::RustAnalyzer { compiler: self.compiler, target: self.target }) - { - install_sh(builder, "rust-analyzer", self.compiler.stage, Some(self.target), &tarball); - } else { - builder.info( - &format!("skipping Install rust-analyzer stage{} ({})", self.compiler.stage, self.target), - ); - } - }; - Clippy, alias = "clippy", Self::should_build(_config), only_hosts: true, { - let tarball = builder - .ensure(dist::Clippy { compiler: self.compiler, target: self.target }) - .expect("missing clippy"); - install_sh(builder, "clippy", self.compiler.stage, Some(self.target), &tarball); - }; - Miri, alias = "miri", Self::should_build(_config), only_hosts: true, { - if let Some(tarball) = builder.ensure(dist::Miri { compiler: self.compiler, target: self.target }) { - install_sh(builder, "miri", self.compiler.stage, Some(self.target), &tarball); - } else { - // Miri is only available on nightly - builder.info( - &format!("skipping Install miri stage{} ({})", self.compiler.stage, self.target), - ); - } - }; - LlvmTools, alias = "llvm-tools", Self::should_build(_config), only_hosts: true, { - if let Some(tarball) = builder.ensure(dist::LlvmTools { target: self.target }) { - install_sh(builder, "llvm-tools", self.compiler.stage, Some(self.target), &tarball); - } else { - builder.info( - &format!("skipping llvm-tools stage{} ({}): external LLVM", self.compiler.stage, self.target), - ); - } - }; - Rustfmt, alias = "rustfmt", Self::should_build(_config), only_hosts: true, { - if let Some(tarball) = builder.ensure(dist::Rustfmt { - compiler: self.compiler, - target: self.target - }) { - install_sh(builder, "rustfmt", self.compiler.stage, Some(self.target), &tarball); - } else { - builder.info( - &format!("skipping Install Rustfmt stage{} ({})", self.compiler.stage, self.target), - ); - } - }; - RustDemangler, alias = "rust-demangler", Self::should_build(_config), only_hosts: true, { - // Note: Even though `should_build` may return true for `extended` default tools, - // dist::RustDemangler may still return None, unless the target-dependent `profiler` config - // is also true, or the `tools` array explicitly includes "rust-demangler". - if let Some(tarball) = builder.ensure(dist::RustDemangler { - compiler: self.compiler, - target: self.target - }) { - install_sh(builder, "rust-demangler", self.compiler.stage, Some(self.target), &tarball); - } else { - builder.info( - &format!("skipping Install RustDemangler stage{} ({})", - self.compiler.stage, self.target), - ); - } - }; - Rustc, path = "compiler/rustc", true, only_hosts: true, { - let tarball = builder.ensure(dist::Rustc { - compiler: builder.compiler(builder.top_stage, self.target), - }); - install_sh(builder, "rustc", self.compiler.stage, Some(self.target), &tarball); - }; -); - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Src { - pub stage: u32, -} - -impl Step for Src { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let config = &run.builder.config; - let cond = config.extended && config.tools.as_ref().map_or(true, |t| t.contains("src")); - run.path("src").default_condition(cond) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Src { stage: run.builder.top_stage }); - } - - fn run(self, builder: &Builder<'_>) { - let tarball = builder.ensure(dist::Src); - install_sh(builder, "src", self.stage, None, &tarball); - } -} diff --git a/src/bootstrap/job.rs b/src/bootstrap/job.rs deleted file mode 100644 index b0a97b540..000000000 --- a/src/bootstrap/job.rs +++ /dev/null @@ -1,143 +0,0 @@ -//! Job management on Windows for bootstrapping -//! -//! Most of the time when you're running a build system (e.g., make) you expect -//! Ctrl-C or abnormal termination to actually terminate the entire tree of -//! process in play, not just the one at the top. This currently works "by -//! default" on Unix platforms because Ctrl-C actually sends a signal to the -//! *process group* rather than the parent process, so everything will get torn -//! down. On Windows, however, this does not happen and Ctrl-C just kills the -//! parent process. -//! -//! To achieve the same semantics on Windows we use Job Objects to ensure that -//! all processes die at the same time. Job objects have a mode of operation -//! where when all handles to the object are closed it causes all child -//! processes associated with the object to be terminated immediately. -//! Conveniently whenever a process in the job object spawns a new process the -//! child will be associated with the job object as well. This means if we add -//! ourselves to the job object we create then everything will get torn down! -//! -//! Unfortunately most of the time the build system is actually called from a -//! python wrapper (which manages things like building the build system) so this -//! all doesn't quite cut it so far. To go the last mile we duplicate the job -//! object handle into our parent process (a python process probably) and then -//! close our own handle. This means that the only handle to the job object -//! resides in the parent python process, so when python dies the whole build -//! system dies (as one would probably expect!). -//! -//! Note that this module has a #[cfg(windows)] above it as none of this logic -//! is required on Unix. - -use crate::Build; -use std::env; -use std::ffi::c_void; -use std::io; -use std::mem; - -use windows::{ - core::PCWSTR, - Win32::Foundation::{CloseHandle, DuplicateHandle, DUPLICATE_SAME_ACCESS, HANDLE}, - Win32::System::Diagnostics::Debug::{SetErrorMode, SEM_NOGPFAULTERRORBOX, THREAD_ERROR_MODE}, - Win32::System::JobObjects::{ - AssignProcessToJobObject, CreateJobObjectW, JobObjectExtendedLimitInformation, - SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, - JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOB_OBJECT_LIMIT_PRIORITY_CLASS, - }, - Win32::System::Threading::{ - GetCurrentProcess, OpenProcess, BELOW_NORMAL_PRIORITY_CLASS, PROCESS_DUP_HANDLE, - }, -}; - -pub unsafe fn setup(build: &mut Build) { - // Enable the Windows Error Reporting dialog which msys disables, - // so we can JIT debug rustc - let mode = SetErrorMode(THREAD_ERROR_MODE::default()); - let mode = THREAD_ERROR_MODE(mode); - SetErrorMode(mode & !SEM_NOGPFAULTERRORBOX); - - // Create a new job object for us to use - let job = CreateJobObjectW(None, PCWSTR::null()).unwrap(); - - // Indicate that when all handles to the job object are gone that all - // process in the object should be killed. Note that this includes our - // entire process tree by default because we've added ourselves and our - // children will reside in the job by default. - let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default(); - info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; - if build.config.low_priority { - info.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_PRIORITY_CLASS; - info.BasicLimitInformation.PriorityClass = BELOW_NORMAL_PRIORITY_CLASS.0; - } - let r = SetInformationJobObject( - job, - JobObjectExtendedLimitInformation, - &info as *const _ as *const c_void, - mem::size_of_val(&info) as u32, - ) - .ok(); - assert!(r.is_ok(), "{}", io::Error::last_os_error()); - - // Assign our process to this job object. Note that if this fails, one very - // likely reason is that we are ourselves already in a job object! This can - // happen on the build bots that we've got for Windows, or if just anyone - // else is instrumenting the build. In this case we just bail out - // immediately and assume that they take care of it. - // - // Also note that nested jobs (why this might fail) are supported in recent - // versions of Windows, but the version of Windows that our bots are running - // at least don't support nested job objects. - let r = AssignProcessToJobObject(job, GetCurrentProcess()).ok(); - if r.is_err() { - CloseHandle(job); - return; - } - - // If we've got a parent process (e.g., the python script that called us) - // then move ownership of this job object up to them. That way if the python - // script is killed (e.g., via ctrl-c) then we'll all be torn down. - // - // If we don't have a parent (e.g., this was run directly) then we - // intentionally leak the job object handle. When our process exits - // (normally or abnormally) it will close the handle implicitly, causing all - // processes in the job to be cleaned up. - let pid = match env::var("BOOTSTRAP_PARENT_ID") { - Ok(s) => s, - Err(..) => return, - }; - - let parent = match OpenProcess(PROCESS_DUP_HANDLE, false, pid.parse().unwrap()).ok() { - Some(parent) => parent, - _ => { - // If we get a null parent pointer here, it is possible that either - // we have an invalid pid or the parent process has been closed. - // Since the first case rarely happens - // (only when wrongly setting the environmental variable), - // it might be better to improve the experience of the second case - // when users have interrupted the parent process and we haven't finish - // duplicating the handle yet. We just need close the job object if that occurs. - CloseHandle(job); - return; - } - }; - - let mut parent_handle = HANDLE::default(); - let r = DuplicateHandle( - GetCurrentProcess(), - job, - parent, - &mut parent_handle, - 0, - false, - DUPLICATE_SAME_ACCESS, - ) - .ok(); - - // If this failed, well at least we tried! An example of DuplicateHandle - // failing in the past has been when the wrong python2 package spawned this - // build system (e.g., the `python2` package in MSYS instead of - // `mingw-w64-x86_64-python2`). Not sure why it failed, but the "failure - // mode" here is that we only clean everything up when the build system - // dies, not when the python parent does, so not too bad. - if r.is_err() { - CloseHandle(job); - } -} diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs deleted file mode 100644 index 8b8d4b237..000000000 --- a/src/bootstrap/lib.rs +++ /dev/null @@ -1,1846 +0,0 @@ -//! Implementation of rustbuild, the Rust build system. -//! -//! This module, and its descendants, are the implementation of the Rust build -//! system. Most of this build system is backed by Cargo but the outer layer -//! here serves as the ability to orchestrate calling Cargo, sequencing Cargo -//! builds, building artifacts like LLVM, etc. The goals of rustbuild are: -//! -//! * To be an easily understandable, easily extensible, and maintainable build -//! system. -//! * Leverage standard tools in the Rust ecosystem to build the compiler, aka -//! crates.io and Cargo. -//! * A standard interface to build across all platforms, including MSVC -//! -//! ## Further information -//! -//! More documentation can be found in each respective module below, and you can -//! also check out the `src/bootstrap/README.md` file for more information. - -use std::cell::{Cell, RefCell}; -use std::collections::{HashMap, HashSet}; -use std::env; -use std::fmt::Display; -use std::fs::{self, File}; -use std::io; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use std::str; - -use build_helper::ci::{gha, CiEnv}; -use build_helper::exit; -use channel::GitInfo; -use config::{DryRun, Target}; -use filetime::FileTime; -use once_cell::sync::OnceCell; - -use crate::builder::Kind; -use crate::config::{LlvmLibunwind, TargetSelection}; -use crate::util::{ - dir_is_empty, exe, libdir, mtime, output, run, run_suppressed, symlink_dir, try_run_suppressed, -}; - -mod builder; -mod cache; -mod cc_detect; -mod channel; -mod check; -mod clean; -mod compile; -mod config; -mod dist; -mod doc; -mod download; -mod flags; -mod format; -mod install; -mod llvm; -mod metadata; -mod render_tests; -mod run; -mod sanity; -mod setup; -mod suggest; -mod synthetic_targets; -mod tarball; -mod test; -mod tool; -mod toolstate; -pub mod util; - -#[cfg(feature = "build-metrics")] -mod metrics; - -#[cfg(windows)] -mod job; - -#[cfg(all(unix, not(target_os = "haiku")))] -mod job { - pub unsafe fn setup(build: &mut crate::Build) { - if build.config.low_priority { - libc::setpriority(libc::PRIO_PGRP as _, 0, 10); - } - } -} - -#[cfg(any(target_os = "haiku", target_os = "hermit", not(any(unix, windows))))] -mod job { - pub unsafe fn setup(_build: &mut crate::Build) {} -} - -pub use crate::builder::PathSet; -use crate::cache::{Interned, INTERNER}; -pub use crate::config::Config; -pub use crate::flags::Subcommand; -use termcolor::{ColorChoice, StandardStream, WriteColor}; - -const LLVM_TOOLS: &[&str] = &[ - "llvm-cov", // used to generate coverage report - "llvm-nm", // used to inspect binaries; it shows symbol names, their sizes and visibility - "llvm-objcopy", // used to transform ELFs into binary format which flashing tools consume - "llvm-objdump", // used to disassemble programs - "llvm-profdata", // used to inspect and merge files generated by profiles - "llvm-readobj", // used to get information from ELFs/objects that the other tools don't provide - "llvm-size", // used to prints the size of the linker sections of a program - "llvm-strip", // used to discard symbols from binary files to reduce their size - "llvm-ar", // used for creating and modifying archive files - "llvm-as", // used to convert LLVM assembly to LLVM bitcode - "llvm-dis", // used to disassemble LLVM bitcode - "llc", // used to compile LLVM bytecode - "opt", // used to optimize LLVM bytecode -]; - -/// LLD file names for all flavors. -const LLD_FILE_NAMES: &[&str] = &["ld.lld", "ld64.lld", "lld-link", "wasm-ld"]; - -pub const VERSION: usize = 2; - -/// Extra --check-cfg to add when building -/// (Mode restriction, config name, config values (if any)) -const EXTRA_CHECK_CFGS: &[(Option, &str, Option<&[&'static str]>)] = &[ - (None, "bootstrap", None), - (Some(Mode::Rustc), "parallel_compiler", None), - (Some(Mode::ToolRustc), "parallel_compiler", None), - (Some(Mode::Codegen), "parallel_compiler", None), - (Some(Mode::Std), "stdarch_intel_sde", None), - (Some(Mode::Std), "no_fp_fmt_parse", None), - (Some(Mode::Std), "no_global_oom_handling", None), - (Some(Mode::Std), "no_rc", None), - (Some(Mode::Std), "no_sync", None), - (Some(Mode::Std), "freebsd12", None), - (Some(Mode::Std), "freebsd13", None), - (Some(Mode::Std), "backtrace_in_libstd", None), - /* Extra values not defined in the built-in targets yet, but used in std */ - // #[cfg(bootstrap)] - (Some(Mode::Std), "target_vendor", Some(&["unikraft"])), - (Some(Mode::Std), "target_env", Some(&["libnx"])), - // #[cfg(bootstrap)] hurd - (Some(Mode::Std), "target_os", Some(&["teeos", "hurd"])), - (Some(Mode::Rustc), "target_os", Some(&["hurd"])), - // #[cfg(bootstrap)] mips32r6, mips64r6 - ( - Some(Mode::Std), - "target_arch", - Some(&["asmjs", "spirv", "nvptx", "xtensa", "mips32r6", "mips64r6", "csky"]), - ), - /* Extra names used by dependencies */ - // FIXME: Used by serde_json, but we should not be triggering on external dependencies. - (Some(Mode::Rustc), "no_btreemap_remove_entry", None), - (Some(Mode::ToolRustc), "no_btreemap_remove_entry", None), - // FIXME: Used by crossbeam-utils, but we should not be triggering on external dependencies. - (Some(Mode::Rustc), "crossbeam_loom", None), - (Some(Mode::ToolRustc), "crossbeam_loom", None), - // FIXME: Used by proc-macro2, but we should not be triggering on external dependencies. - (Some(Mode::Rustc), "span_locations", None), - (Some(Mode::ToolRustc), "span_locations", None), - // FIXME: Used by rustix, but we should not be triggering on external dependencies. - (Some(Mode::Rustc), "rustix_use_libc", None), - (Some(Mode::ToolRustc), "rustix_use_libc", None), - // FIXME: Used by filetime, but we should not be triggering on external dependencies. - (Some(Mode::Rustc), "emulate_second_only_system", None), - (Some(Mode::ToolRustc), "emulate_second_only_system", None), - // Needed to avoid the need to copy windows.lib into the sysroot. - (Some(Mode::Rustc), "windows_raw_dylib", None), - (Some(Mode::ToolRustc), "windows_raw_dylib", None), -]; - -/// A structure representing a Rust compiler. -/// -/// Each compiler has a `stage` that it is associated with and a `host` that -/// corresponds to the platform the compiler runs on. This structure is used as -/// a parameter to many methods below. -#[derive(Eq, PartialOrd, Ord, PartialEq, Clone, Copy, Hash, Debug)] -pub struct Compiler { - stage: u32, - host: TargetSelection, -} - -#[derive(PartialEq, Eq, Copy, Clone, Debug)] -pub enum DocTests { - /// Run normal tests and doc tests (default). - Yes, - /// Do not run any doc tests. - No, - /// Only run doc tests. - Only, -} - -pub enum GitRepo { - Rustc, - Llvm, -} - -/// Global configuration for the build system. -/// -/// This structure transitively contains all configuration for the build system. -/// All filesystem-encoded configuration is in `config`, all flags are in -/// `flags`, and then parsed or probed information is listed in the keys below. -/// -/// This structure is a parameter of almost all methods in the build system, -/// although most functions are implemented as free functions rather than -/// methods specifically on this structure itself (to make it easier to -/// organize). -#[derive(Clone)] -pub struct Build { - /// User-specified configuration from `config.toml`. - config: Config, - - // Version information - version: String, - - // Properties derived from the above configuration - src: PathBuf, - out: PathBuf, - bootstrap_out: PathBuf, - cargo_info: channel::GitInfo, - rust_analyzer_info: channel::GitInfo, - clippy_info: channel::GitInfo, - miri_info: channel::GitInfo, - rustfmt_info: channel::GitInfo, - in_tree_llvm_info: channel::GitInfo, - local_rebuild: bool, - fail_fast: bool, - doc_tests: DocTests, - verbosity: usize, - - // Targets for which to build - build: TargetSelection, - hosts: Vec, - targets: Vec, - - initial_rustc: PathBuf, - initial_cargo: PathBuf, - initial_lld: PathBuf, - initial_libdir: PathBuf, - initial_sysroot: PathBuf, - - // Runtime state filled in later on - // C/C++ compilers and archiver for all targets - cc: RefCell>, - cxx: RefCell>, - ar: RefCell>, - ranlib: RefCell>, - // Miscellaneous - // allow bidirectional lookups: both name -> path and path -> name - crates: HashMap, Crate>, - crate_paths: HashMap>, - is_sudo: bool, - ci_env: CiEnv, - delayed_failures: RefCell>, - prerelease_version: Cell>, - - #[cfg(feature = "build-metrics")] - metrics: metrics::BuildMetrics, -} - -#[derive(Debug, Clone)] -struct Crate { - name: Interned, - deps: HashSet>, - path: PathBuf, - has_lib: bool, -} - -impl Crate { - fn local_path(&self, build: &Build) -> PathBuf { - self.path.strip_prefix(&build.config.src).unwrap().into() - } -} - -/// When building Rust various objects are handled differently. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub enum DependencyType { - /// Libraries originating from proc-macros. - Host, - /// Typical Rust libraries. - Target, - /// Non Rust libraries and objects shipped to ease usage of certain targets. - TargetSelfContained, -} - -/// The various "modes" of invoking Cargo. -/// -/// These entries currently correspond to the various output directories of the -/// build system, with each mod generating output in a different directory. -#[derive(Debug, Hash, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub enum Mode { - /// Build the standard library, placing output in the "stageN-std" directory. - Std, - - /// Build librustc, and compiler libraries, placing output in the "stageN-rustc" directory. - Rustc, - - /// Build a codegen backend for rustc, placing the output in the "stageN-codegen" directory. - Codegen, - - /// Build a tool, placing output in the "stage0-bootstrap-tools" - /// directory. This is for miscellaneous sets of tools that are built - /// using the bootstrap stage0 compiler in its entirety (target libraries - /// and all). Typically these tools compile with stable Rust. - ToolBootstrap, - - /// Build a tool which uses the locally built std, placing output in the - /// "stageN-tools" directory. Its usage is quite rare, mainly used by - /// compiletest which needs libtest. - ToolStd, - - /// Build a tool which uses the locally built rustc and the target std, - /// placing the output in the "stageN-tools" directory. This is used for - /// anything that needs a fully functional rustc, such as rustdoc, clippy, - /// cargo, rls, rustfmt, miri, etc. - ToolRustc, -} - -impl Mode { - pub fn is_tool(&self) -> bool { - matches!(self, Mode::ToolBootstrap | Mode::ToolRustc | Mode::ToolStd) - } - - pub fn must_support_dlopen(&self) -> bool { - matches!(self, Mode::Std | Mode::Codegen) - } -} - -pub enum CLang { - C, - Cxx, -} - -macro_rules! forward { - ( $( $fn:ident( $($param:ident: $ty:ty),* ) $( -> $ret:ty)? ),+ $(,)? ) => { - impl Build { - $( fn $fn(&self, $($param: $ty),* ) $( -> $ret)? { - self.config.$fn( $($param),* ) - } )+ - } - } -} - -forward! { - verbose(msg: &str), - is_verbose() -> bool, - create(path: &Path, s: &str), - remove(f: &Path), - tempdir() -> PathBuf, - llvm_link_shared() -> bool, - download_rustc() -> bool, - initial_rustfmt() -> Option, -} - -impl Build { - /// Creates a new set of build configuration from the `flags` on the command - /// line and the filesystem `config`. - /// - /// By default all build output will be placed in the current directory. - pub fn new(mut config: Config) -> Build { - let src = config.src.clone(); - let out = config.out.clone(); - - #[cfg(unix)] - // keep this consistent with the equivalent check in x.py: - // https://github.com/rust-lang/rust/blob/a8a33cf27166d3eabaffc58ed3799e054af3b0c6/src/bootstrap/bootstrap.py#L796-L797 - let is_sudo = match env::var_os("SUDO_USER") { - Some(_sudo_user) => { - let uid = unsafe { libc::getuid() }; - uid == 0 - } - None => false, - }; - #[cfg(not(unix))] - let is_sudo = false; - - let omit_git_hash = config.omit_git_hash; - let rust_info = channel::GitInfo::new(omit_git_hash, &src); - let cargo_info = channel::GitInfo::new(omit_git_hash, &src.join("src/tools/cargo")); - let rust_analyzer_info = - channel::GitInfo::new(omit_git_hash, &src.join("src/tools/rust-analyzer")); - let clippy_info = channel::GitInfo::new(omit_git_hash, &src.join("src/tools/clippy")); - let miri_info = channel::GitInfo::new(omit_git_hash, &src.join("src/tools/miri")); - let rustfmt_info = channel::GitInfo::new(omit_git_hash, &src.join("src/tools/rustfmt")); - - // we always try to use git for LLVM builds - let in_tree_llvm_info = channel::GitInfo::new(false, &src.join("src/llvm-project")); - - let initial_target_libdir_str = if config.dry_run() { - "/dummy/lib/path/to/lib/".to_string() - } else { - output( - Command::new(&config.initial_rustc) - .arg("--target") - .arg(config.build.rustc_target_arg()) - .arg("--print") - .arg("target-libdir"), - ) - }; - let initial_target_dir = Path::new(&initial_target_libdir_str).parent().unwrap(); - let initial_lld = initial_target_dir.join("bin").join("rust-lld"); - - let initial_sysroot = if config.dry_run() { - "/dummy".to_string() - } else { - output(Command::new(&config.initial_rustc).arg("--print").arg("sysroot")) - } - .trim() - .to_string(); - - let initial_libdir = initial_target_dir - .parent() - .unwrap() - .parent() - .unwrap() - .strip_prefix(&initial_sysroot) - .unwrap() - .to_path_buf(); - - let version = std::fs::read_to_string(src.join("src").join("version")) - .expect("failed to read src/version"); - let version = version.trim(); - - let bootstrap_out = std::env::current_exe() - .expect("could not determine path to running process") - .parent() - .unwrap() - .to_path_buf(); - if !bootstrap_out.join(exe("rustc", config.build)).exists() && !cfg!(test) { - // this restriction can be lifted whenever https://github.com/rust-lang/rfcs/pull/3028 is implemented - panic!( - "`rustc` not found in {}, run `cargo build --bins` before `cargo run`", - bootstrap_out.display() - ) - } - - if rust_info.is_from_tarball() && config.description.is_none() { - config.description = Some("built from a source tarball".to_owned()); - } - - let mut build = Build { - initial_rustc: config.initial_rustc.clone(), - initial_cargo: config.initial_cargo.clone(), - initial_lld, - initial_libdir, - initial_sysroot: initial_sysroot.into(), - local_rebuild: config.local_rebuild, - fail_fast: config.cmd.fail_fast(), - doc_tests: config.cmd.doc_tests(), - verbosity: config.verbose, - - build: config.build, - hosts: config.hosts.clone(), - targets: config.targets.clone(), - - config, - version: version.to_string(), - src, - out, - bootstrap_out, - - cargo_info, - rust_analyzer_info, - clippy_info, - miri_info, - rustfmt_info, - in_tree_llvm_info, - cc: RefCell::new(HashMap::new()), - cxx: RefCell::new(HashMap::new()), - ar: RefCell::new(HashMap::new()), - ranlib: RefCell::new(HashMap::new()), - crates: HashMap::new(), - crate_paths: HashMap::new(), - is_sudo, - ci_env: CiEnv::current(), - delayed_failures: RefCell::new(Vec::new()), - prerelease_version: Cell::new(None), - - #[cfg(feature = "build-metrics")] - metrics: metrics::BuildMetrics::init(), - }; - - // If local-rust is the same major.minor as the current version, then force a - // local-rebuild - let local_version_verbose = - output(Command::new(&build.initial_rustc).arg("--version").arg("--verbose")); - let local_release = local_version_verbose - .lines() - .filter_map(|x| x.strip_prefix("release:")) - .next() - .unwrap() - .trim(); - if local_release.split('.').take(2).eq(version.split('.').take(2)) { - build.verbose(&format!("auto-detected local-rebuild {local_release}")); - build.local_rebuild = true; - } - - build.verbose("finding compilers"); - cc_detect::find(&build); - // When running `setup`, the profile is about to change, so any requirements we have now may - // be different on the next invocation. Don't check for them until the next time x.py is - // run. This is ok because `setup` never runs any build commands, so it won't fail if commands are missing. - // - // Similarly, for `setup` we don't actually need submodules or cargo metadata. - if !matches!(build.config.cmd, Subcommand::Setup { .. }) { - build.verbose("running sanity check"); - sanity::check(&mut build); - - // Make sure we update these before gathering metadata so we don't get an error about missing - // Cargo.toml files. - let rust_submodules = ["src/tools/cargo", "library/backtrace", "library/stdarch"]; - for s in rust_submodules { - build.update_submodule(Path::new(s)); - } - // Now, update all existing submodules. - build.update_existing_submodules(); - - build.verbose("learning about cargo"); - metadata::build(&mut build); - } - - // Make a symbolic link so we can use a consistent directory in the documentation. - let build_triple = build.out.join(&build.build.triple); - t!(fs::create_dir_all(&build_triple)); - let host = build.out.join("host"); - if host.is_symlink() { - // Left over from a previous build; overwrite it. - // This matters if `build.build` has changed between invocations. - #[cfg(windows)] - t!(fs::remove_dir(&host)); - #[cfg(not(windows))] - t!(fs::remove_file(&host)); - } - t!( - symlink_dir(&build.config, &build_triple, &host), - format!("symlink_dir({} => {}) failed", host.display(), build_triple.display()) - ); - - build - } - - // modified from `check_submodule` and `update_submodule` in bootstrap.py - /// Given a path to the directory of a submodule, update it. - /// - /// `relative_path` should be relative to the root of the git repository, not an absolute path. - pub(crate) fn update_submodule(&self, relative_path: &Path) { - if !self.config.submodules(&self.rust_info()) { - return; - } - - let absolute_path = self.config.src.join(relative_path); - - // NOTE: The check for the empty directory is here because when running x.py the first time, - // the submodule won't be checked out. Check it out now so we can build it. - if !channel::GitInfo::new(false, &absolute_path).is_managed_git_subrepository() - && !dir_is_empty(&absolute_path) - { - return; - } - - // check_submodule - let checked_out_hash = - output(Command::new("git").args(&["rev-parse", "HEAD"]).current_dir(&absolute_path)); - // update_submodules - let recorded = output( - Command::new("git") - .args(&["ls-tree", "HEAD"]) - .arg(relative_path) - .current_dir(&self.config.src), - ); - let actual_hash = recorded - .split_whitespace() - .nth(2) - .unwrap_or_else(|| panic!("unexpected output `{}`", recorded)); - - // update_submodule - if actual_hash == checked_out_hash.trim_end() { - // already checked out - return; - } - - println!("Updating submodule {}", relative_path.display()); - self.run( - Command::new("git") - .args(&["submodule", "-q", "sync"]) - .arg(relative_path) - .current_dir(&self.config.src), - ); - - // Try passing `--progress` to start, then run git again without if that fails. - let update = |progress: bool| { - // Git is buggy and will try to fetch submodules from the tracking branch for *this* repository, - // even though that has no relation to the upstream for the submodule. - let current_branch = { - let output = self - .config - .git() - .args(["symbolic-ref", "--short", "HEAD"]) - .stderr(Stdio::inherit()) - .output(); - let output = t!(output); - if output.status.success() { - Some(String::from_utf8(output.stdout).unwrap().trim().to_owned()) - } else { - None - } - }; - - let mut git = self.config.git(); - if let Some(branch) = current_branch { - // If there is a tag named after the current branch, git will try to disambiguate by prepending `heads/` to the branch name. - // This syntax isn't accepted by `branch.{branch}`. Strip it. - let branch = branch.strip_prefix("heads/").unwrap_or(&branch); - git.arg("-c").arg(format!("branch.{branch}.remote=origin")); - } - git.args(&["submodule", "update", "--init", "--recursive", "--depth=1"]); - if progress { - git.arg("--progress"); - } - git.arg(relative_path); - git - }; - // NOTE: doesn't use `try_run` because this shouldn't print an error if it fails. - if !update(true).status().map_or(false, |status| status.success()) { - self.run(&mut update(false)); - } - - // Save any local changes, but avoid running `git stash pop` if there are none (since it will exit with an error). - #[allow(deprecated)] // diff-index reports the modifications through the exit status - let has_local_modifications = self - .config - .try_run( - Command::new("git") - .args(&["diff-index", "--quiet", "HEAD"]) - .current_dir(&absolute_path), - ) - .is_err(); - if has_local_modifications { - self.run(Command::new("git").args(&["stash", "push"]).current_dir(&absolute_path)); - } - - self.run(Command::new("git").args(&["reset", "-q", "--hard"]).current_dir(&absolute_path)); - self.run(Command::new("git").args(&["clean", "-qdfx"]).current_dir(&absolute_path)); - - if has_local_modifications { - self.run(Command::new("git").args(&["stash", "pop"]).current_dir(absolute_path)); - } - } - - /// If any submodule has been initialized already, sync it unconditionally. - /// This avoids contributors checking in a submodule change by accident. - pub fn update_existing_submodules(&self) { - // Avoid running git when there isn't a git checkout. - if !self.config.submodules(&self.rust_info()) { - return; - } - let output = output( - self.config - .git() - .args(&["config", "--file"]) - .arg(&self.config.src.join(".gitmodules")) - .args(&["--get-regexp", "path"]), - ); - for line in output.lines() { - // Look for `submodule.$name.path = $path` - // Sample output: `submodule.src/rust-installer.path src/tools/rust-installer` - let submodule = Path::new(line.splitn(2, ' ').nth(1).unwrap()); - // Don't update the submodule unless it's already been cloned. - if channel::GitInfo::new(false, submodule).is_managed_git_subrepository() { - self.update_submodule(submodule); - } - } - } - - /// Executes the entire build, as configured by the flags and configuration. - pub fn build(&mut self) { - unsafe { - job::setup(self); - } - - // Download rustfmt early so that it can be used in rust-analyzer configs. - let _ = &builder::Builder::new(&self).initial_rustfmt(); - - // hardcoded subcommands - match &self.config.cmd { - Subcommand::Format { check } => { - return format::format(&builder::Builder::new(&self), *check, &self.config.paths); - } - Subcommand::Suggest { run } => { - return suggest::suggest(&builder::Builder::new(&self), *run); - } - _ => (), - } - - { - let builder = builder::Builder::new(&self); - if let Some(path) = builder.paths.get(0) { - if path == Path::new("nonexistent/path/to/trigger/cargo/metadata") { - return; - } - } - } - - if !self.config.dry_run() { - { - self.config.dry_run = DryRun::SelfCheck; - let builder = builder::Builder::new(&self); - builder.execute_cli(); - } - self.config.dry_run = DryRun::Disabled; - let builder = builder::Builder::new(&self); - builder.execute_cli(); - } else { - let builder = builder::Builder::new(&self); - builder.execute_cli(); - } - - // Check for postponed failures from `test --no-fail-fast`. - let failures = self.delayed_failures.borrow(); - if failures.len() > 0 { - eprintln!("\n{} command(s) did not execute successfully:\n", failures.len()); - for failure in failures.iter() { - eprintln!(" - {failure}\n"); - } - exit!(1); - } - - #[cfg(feature = "build-metrics")] - self.metrics.persist(self); - } - - /// Clear out `dir` if `input` is newer. - /// - /// After this executes, it will also ensure that `dir` exists. - fn clear_if_dirty(&self, dir: &Path, input: &Path) -> bool { - let stamp = dir.join(".stamp"); - let mut cleared = false; - if mtime(&stamp) < mtime(input) { - self.verbose(&format!("Dirty - {}", dir.display())); - let _ = fs::remove_dir_all(dir); - cleared = true; - } else if stamp.exists() { - return cleared; - } - t!(fs::create_dir_all(dir)); - t!(File::create(stamp)); - cleared - } - - fn rust_info(&self) -> &GitInfo { - &self.config.rust_info - } - - /// Gets the space-separated set of activated features for the standard - /// library. - fn std_features(&self, target: TargetSelection) -> String { - let mut features = " panic-unwind".to_string(); - - match self.config.llvm_libunwind(target) { - LlvmLibunwind::InTree => features.push_str(" llvm-libunwind"), - LlvmLibunwind::System => features.push_str(" system-llvm-libunwind"), - LlvmLibunwind::No => {} - } - if self.config.backtrace { - features.push_str(" backtrace"); - } - if self.config.profiler_enabled(target) { - features.push_str(" profiler"); - } - features - } - - /// Gets the space-separated set of activated features for the compiler. - fn rustc_features(&self, kind: Kind) -> String { - let mut features = vec![]; - if self.config.jemalloc { - features.push("jemalloc"); - } - if self.config.llvm_enabled() || kind == Kind::Check { - features.push("llvm"); - } - // keep in sync with `bootstrap/compile.rs:rustc_cargo_env` - if self.config.rustc_parallel { - features.push("rustc_use_parallel_compiler"); - } - - // If debug logging is on, then we want the default for tracing: - // https://github.com/tokio-rs/tracing/blob/3dd5c03d907afdf2c39444a29931833335171554/tracing/src/level_filters.rs#L26 - // which is everything (including debug/trace/etc.) - // if its unset, if debug_assertions is on, then debug_logging will also be on - // as well as tracing *ignoring* this feature when debug_assertions is on - if !self.config.rust_debug_logging { - features.push("max_level_info"); - } - - features.join(" ") - } - - /// Component directory that Cargo will produce output into (e.g. - /// release/debug) - fn cargo_dir(&self) -> &'static str { - if self.config.rust_optimize.is_release() { "release" } else { "debug" } - } - - fn tools_dir(&self, compiler: Compiler) -> PathBuf { - let out = self - .out - .join(&*compiler.host.triple) - .join(format!("stage{}-tools-bin", compiler.stage)); - t!(fs::create_dir_all(&out)); - out - } - - /// Returns the root directory for all output generated in a particular - /// stage when running with a particular host compiler. - /// - /// The mode indicates what the root directory is for. - fn stage_out(&self, compiler: Compiler, mode: Mode) -> PathBuf { - let suffix = match mode { - Mode::Std => "-std", - Mode::Rustc => "-rustc", - Mode::Codegen => "-codegen", - Mode::ToolBootstrap => "-bootstrap-tools", - Mode::ToolStd | Mode::ToolRustc => "-tools", - }; - self.out.join(&*compiler.host.triple).join(format!("stage{}{}", compiler.stage, suffix)) - } - - /// Returns the root output directory for all Cargo output in a given stage, - /// running a particular compiler, whether or not we're building the - /// standard library, and targeting the specified architecture. - fn cargo_out(&self, compiler: Compiler, mode: Mode, target: TargetSelection) -> PathBuf { - self.stage_out(compiler, mode).join(&*target.triple).join(self.cargo_dir()) - } - - /// Root output directory for LLVM compiled for `target` - /// - /// Note that if LLVM is configured externally then the directory returned - /// will likely be empty. - fn llvm_out(&self, target: TargetSelection) -> PathBuf { - self.out.join(&*target.triple).join("llvm") - } - - fn lld_out(&self, target: TargetSelection) -> PathBuf { - self.out.join(&*target.triple).join("lld") - } - - /// Output directory for all documentation for a target - fn doc_out(&self, target: TargetSelection) -> PathBuf { - self.out.join(&*target.triple).join("doc") - } - - /// Output directory for all JSON-formatted documentation for a target - fn json_doc_out(&self, target: TargetSelection) -> PathBuf { - self.out.join(&*target.triple).join("json-doc") - } - - fn test_out(&self, target: TargetSelection) -> PathBuf { - self.out.join(&*target.triple).join("test") - } - - /// Output directory for all documentation for a target - fn compiler_doc_out(&self, target: TargetSelection) -> PathBuf { - self.out.join(&*target.triple).join("compiler-doc") - } - - /// Output directory for some generated md crate documentation for a target (temporary) - fn md_doc_out(&self, target: TargetSelection) -> Interned { - INTERNER.intern_path(self.out.join(&*target.triple).join("md-doc")) - } - - /// Returns `true` if no custom `llvm-config` is set for the specified target. - /// - /// If no custom `llvm-config` was specified then Rust's llvm will be used. - fn is_rust_llvm(&self, target: TargetSelection) -> bool { - match self.config.target_config.get(&target) { - Some(Target { llvm_has_rust_patches: Some(patched), .. }) => *patched, - Some(Target { llvm_config, .. }) => { - // If the user set llvm-config we assume Rust is not patched, - // but first check to see if it was configured by llvm-from-ci. - (self.config.llvm_from_ci && target == self.config.build) || llvm_config.is_none() - } - None => true, - } - } - - /// Returns the path to `FileCheck` binary for the specified target - fn llvm_filecheck(&self, target: TargetSelection) -> PathBuf { - let target_config = self.config.target_config.get(&target); - if let Some(s) = target_config.and_then(|c| c.llvm_filecheck.as_ref()) { - s.to_path_buf() - } else if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) { - let llvm_bindir = output(Command::new(s).arg("--bindir")); - let filecheck = Path::new(llvm_bindir.trim()).join(exe("FileCheck", target)); - if filecheck.exists() { - filecheck - } else { - // On Fedora the system LLVM installs FileCheck in the - // llvm subdirectory of the libdir. - let llvm_libdir = output(Command::new(s).arg("--libdir")); - let lib_filecheck = - Path::new(llvm_libdir.trim()).join("llvm").join(exe("FileCheck", target)); - if lib_filecheck.exists() { - lib_filecheck - } else { - // Return the most normal file name, even though - // it doesn't exist, so that any error message - // refers to that. - filecheck - } - } - } else { - let base = self.llvm_out(target).join("build"); - let base = if !self.ninja() && target.contains("msvc") { - if self.config.llvm_optimize { - if self.config.llvm_release_debuginfo { - base.join("RelWithDebInfo") - } else { - base.join("Release") - } - } else { - base.join("Debug") - } - } else { - base - }; - base.join("bin").join(exe("FileCheck", target)) - } - } - - /// Directory for libraries built from C/C++ code and shared between stages. - fn native_dir(&self, target: TargetSelection) -> PathBuf { - self.out.join(&*target.triple).join("native") - } - - /// Root output directory for rust_test_helpers library compiled for - /// `target` - fn test_helpers_out(&self, target: TargetSelection) -> PathBuf { - self.native_dir(target).join("rust-test-helpers") - } - - /// Adds the `RUST_TEST_THREADS` env var if necessary - fn add_rust_test_threads(&self, cmd: &mut Command) { - if env::var_os("RUST_TEST_THREADS").is_none() { - cmd.env("RUST_TEST_THREADS", self.jobs().to_string()); - } - } - - /// Returns the libdir of the snapshot compiler. - fn rustc_snapshot_libdir(&self) -> PathBuf { - self.rustc_snapshot_sysroot().join(libdir(self.config.build)) - } - - /// Returns the sysroot of the snapshot compiler. - fn rustc_snapshot_sysroot(&self) -> &Path { - static SYSROOT_CACHE: OnceCell = once_cell::sync::OnceCell::new(); - SYSROOT_CACHE.get_or_init(|| { - let mut rustc = Command::new(&self.initial_rustc); - rustc.args(&["--print", "sysroot"]); - output(&mut rustc).trim().into() - }) - } - - /// Runs a command, printing out nice contextual information if it fails. - fn run(&self, cmd: &mut Command) { - if self.config.dry_run() { - return; - } - self.verbose(&format!("running: {cmd:?}")); - run(cmd, self.is_verbose()) - } - - /// Runs a command, printing out nice contextual information if it fails. - fn run_quiet(&self, cmd: &mut Command) { - if self.config.dry_run() { - return; - } - self.verbose(&format!("running: {cmd:?}")); - run_suppressed(cmd) - } - - /// Runs a command, printing out nice contextual information if it fails. - /// Exits if the command failed to execute at all, otherwise returns its - /// `status.success()`. - fn run_quiet_delaying_failure(&self, cmd: &mut Command) -> bool { - if self.config.dry_run() { - return true; - } - if !self.fail_fast { - self.verbose(&format!("running: {cmd:?}")); - if !try_run_suppressed(cmd) { - let mut failures = self.delayed_failures.borrow_mut(); - failures.push(format!("{cmd:?}")); - return false; - } - } else { - self.run_quiet(cmd); - } - true - } - - /// Runs a command, printing out contextual info if it fails, and delaying errors until the build finishes. - pub(crate) fn run_delaying_failure(&self, cmd: &mut Command) -> bool { - if !self.fail_fast { - #[allow(deprecated)] // can't use Build::try_run, that's us - if self.config.try_run(cmd).is_err() { - let mut failures = self.delayed_failures.borrow_mut(); - failures.push(format!("{cmd:?}")); - return false; - } - } else { - self.run(cmd); - } - true - } - - pub fn is_verbose_than(&self, level: usize) -> bool { - self.verbosity > level - } - - /// Prints a message if this build is configured in more verbose mode than `level`. - fn verbose_than(&self, level: usize, msg: &str) { - if self.is_verbose_than(level) { - println!("{msg}"); - } - } - - fn info(&self, msg: &str) { - match self.config.dry_run { - DryRun::SelfCheck => (), - DryRun::Disabled | DryRun::UserSelected => { - println!("{msg}"); - } - } - } - - #[must_use = "Groups should not be dropped until the Step finishes running"] - #[track_caller] - fn msg_check( - &self, - what: impl Display, - target: impl Into>, - ) -> Option { - self.msg(Kind::Check, self.config.stage, what, self.config.build, target) - } - - #[must_use = "Groups should not be dropped until the Step finishes running"] - #[track_caller] - fn msg_doc( - &self, - compiler: Compiler, - what: impl Display, - target: impl Into> + Copy, - ) -> Option { - self.msg(Kind::Doc, compiler.stage, what, compiler.host, target.into()) - } - - #[must_use = "Groups should not be dropped until the Step finishes running"] - #[track_caller] - fn msg_build( - &self, - compiler: Compiler, - what: impl Display, - target: impl Into>, - ) -> Option { - self.msg(Kind::Build, compiler.stage, what, compiler.host, target) - } - - /// Return a `Group` guard for a [`Step`] that is built for each `--stage`. - /// - /// [`Step`]: crate::builder::Step - #[must_use = "Groups should not be dropped until the Step finishes running"] - #[track_caller] - fn msg( - &self, - action: impl Into, - stage: u32, - what: impl Display, - host: impl Into>, - target: impl Into>, - ) -> Option { - let action = action.into().description(); - let msg = |fmt| format!("{action} stage{stage} {what}{fmt}"); - let msg = if let Some(target) = target.into() { - let host = host.into().unwrap(); - if host == target { - msg(format_args!(" ({target})")) - } else { - msg(format_args!(" ({host} -> {target})")) - } - } else { - msg(format_args!("")) - }; - self.group(&msg) - } - - /// Return a `Group` guard for a [`Step`] that is only built once and isn't affected by `--stage`. - /// - /// [`Step`]: crate::builder::Step - #[must_use = "Groups should not be dropped until the Step finishes running"] - #[track_caller] - fn msg_unstaged( - &self, - action: impl Into, - what: impl Display, - target: TargetSelection, - ) -> Option { - let action = action.into().description(); - let msg = format!("{action} {what} for {target}"); - self.group(&msg) - } - - #[must_use = "Groups should not be dropped until the Step finishes running"] - #[track_caller] - fn msg_sysroot_tool( - &self, - action: impl Into, - stage: u32, - what: impl Display, - host: TargetSelection, - target: TargetSelection, - ) -> Option { - let action = action.into().description(); - let msg = |fmt| format!("{action} {what} {fmt}"); - let msg = if host == target { - msg(format_args!("(stage{stage} -> stage{}, {target})", stage + 1)) - } else { - msg(format_args!("(stage{stage}:{host} -> stage{}:{target})", stage + 1)) - }; - self.group(&msg) - } - - #[track_caller] - fn group(&self, msg: &str) -> Option { - match self.config.dry_run { - DryRun::SelfCheck => None, - DryRun::Disabled | DryRun::UserSelected => Some(gha::group(&msg)), - } - } - - /// Returns the number of parallel jobs that have been configured for this - /// build. - fn jobs(&self) -> u32 { - self.config.jobs.unwrap_or_else(|| { - std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32 - }) - } - - fn debuginfo_map_to(&self, which: GitRepo) -> Option { - if !self.config.rust_remap_debuginfo { - return None; - } - - match which { - GitRepo::Rustc => { - let sha = self.rust_sha().unwrap_or(&self.version); - Some(format!("/rustc/{sha}")) - } - GitRepo::Llvm => Some(String::from("/rustc/llvm")), - } - } - - /// Returns the path to the C compiler for the target specified. - fn cc(&self, target: TargetSelection) -> PathBuf { - if self.config.dry_run() { - return PathBuf::new(); - } - self.cc.borrow()[&target].path().into() - } - - /// Returns a list of flags to pass to the C compiler for the target - /// specified. - fn cflags(&self, target: TargetSelection, which: GitRepo, c: CLang) -> Vec { - if self.config.dry_run() { - return Vec::new(); - } - let base = match c { - CLang::C => self.cc.borrow()[&target].clone(), - CLang::Cxx => self.cxx.borrow()[&target].clone(), - }; - - // Filter out -O and /O (the optimization flags) that we picked up from - // cc-rs because the build scripts will determine that for themselves. - let mut base = base - .args() - .iter() - .map(|s| s.to_string_lossy().into_owned()) - .filter(|s| !s.starts_with("-O") && !s.starts_with("/O")) - .collect::>(); - - // If we're compiling on macOS then we add a few unconditional flags - // indicating that we want libc++ (more filled out than libstdc++) and - // we want to compile for 10.7. This way we can ensure that - // LLVM/etc are all properly compiled. - if target.contains("apple-darwin") { - base.push("-stdlib=libc++".into()); - } - - // Work around an apparently bad MinGW / GCC optimization, - // See: https://lists.llvm.org/pipermail/cfe-dev/2016-December/051980.html - // See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=78936 - if &*target.triple == "i686-pc-windows-gnu" { - base.push("-fno-omit-frame-pointer".into()); - } - - if let Some(map_to) = self.debuginfo_map_to(which) { - let map = format!("{}={}", self.src.display(), map_to); - let cc = self.cc(target); - if cc.ends_with("clang") || cc.ends_with("gcc") { - base.push(format!("-fdebug-prefix-map={map}")); - } else if cc.ends_with("clang-cl.exe") { - base.push("-Xclang".into()); - base.push(format!("-fdebug-prefix-map={map}")); - } - } - base - } - - /// Returns the path to the `ar` archive utility for the target specified. - fn ar(&self, target: TargetSelection) -> Option { - if self.config.dry_run() { - return None; - } - self.ar.borrow().get(&target).cloned() - } - - /// Returns the path to the `ranlib` utility for the target specified. - fn ranlib(&self, target: TargetSelection) -> Option { - if self.config.dry_run() { - return None; - } - self.ranlib.borrow().get(&target).cloned() - } - - /// Returns the path to the C++ compiler for the target specified. - fn cxx(&self, target: TargetSelection) -> Result { - if self.config.dry_run() { - return Ok(PathBuf::new()); - } - match self.cxx.borrow().get(&target) { - Some(p) => Ok(p.path().into()), - None => Err(format!("target `{target}` is not configured as a host, only as a target")), - } - } - - /// Returns the path to the linker for the given target if it needs to be overridden. - fn linker(&self, target: TargetSelection) -> Option { - if self.config.dry_run() { - return Some(PathBuf::new()); - } - if let Some(linker) = self.config.target_config.get(&target).and_then(|c| c.linker.clone()) - { - Some(linker) - } else if target.contains("vxworks") { - // need to use CXX compiler as linker to resolve the exception functions - // that are only existed in CXX libraries - Some(self.cxx.borrow()[&target].path().into()) - } else if target != self.config.build - && util::use_host_linker(target) - && !target.contains("msvc") - { - Some(self.cc(target)) - } else if self.config.use_lld && !self.is_fuse_ld_lld(target) && self.build == target { - Some(self.initial_lld.clone()) - } else { - None - } - } - - // LLD is used through `-fuse-ld=lld` rather than directly. - // Only MSVC targets use LLD directly at the moment. - fn is_fuse_ld_lld(&self, target: TargetSelection) -> bool { - self.config.use_lld && !target.contains("msvc") - } - - fn lld_flags(&self, target: TargetSelection) -> impl Iterator { - let mut options = [None, None]; - - if self.config.use_lld { - if self.is_fuse_ld_lld(target) { - options[0] = Some("-Clink-arg=-fuse-ld=lld".to_string()); - } - - let no_threads = util::lld_flag_no_threads(target.contains("windows")); - options[1] = Some(format!("-Clink-arg=-Wl,{no_threads}")); - } - - IntoIterator::into_iter(options).flatten() - } - - /// Returns if this target should statically link the C runtime, if specified - fn crt_static(&self, target: TargetSelection) -> Option { - if target.contains("pc-windows-msvc") { - Some(true) - } else { - self.config.target_config.get(&target).and_then(|t| t.crt_static) - } - } - - /// Returns the "musl root" for this `target`, if defined - fn musl_root(&self, target: TargetSelection) -> Option<&Path> { - self.config - .target_config - .get(&target) - .and_then(|t| t.musl_root.as_ref()) - .or_else(|| self.config.musl_root.as_ref()) - .map(|p| &**p) - } - - /// Returns the "musl libdir" for this `target`. - fn musl_libdir(&self, target: TargetSelection) -> Option { - let t = self.config.target_config.get(&target)?; - if let libdir @ Some(_) = &t.musl_libdir { - return libdir.clone(); - } - self.musl_root(target).map(|root| root.join("lib")) - } - - /// Returns the sysroot for the wasi target, if defined - fn wasi_root(&self, target: TargetSelection) -> Option<&Path> { - self.config.target_config.get(&target).and_then(|t| t.wasi_root.as_ref()).map(|p| &**p) - } - - /// Returns `true` if this is a no-std `target`, if defined - fn no_std(&self, target: TargetSelection) -> Option { - self.config.target_config.get(&target).map(|t| t.no_std) - } - - /// Returns `true` if the target will be tested using the `remote-test-client` - /// and `remote-test-server` binaries. - fn remote_tested(&self, target: TargetSelection) -> bool { - self.qemu_rootfs(target).is_some() - || target.contains("android") - || env::var_os("TEST_DEVICE_ADDR").is_some() - } - - /// Returns the root of the "rootfs" image that this target will be using, - /// if one was configured. - /// - /// If `Some` is returned then that means that tests for this target are - /// emulated with QEMU and binaries will need to be shipped to the emulator. - fn qemu_rootfs(&self, target: TargetSelection) -> Option<&Path> { - self.config.target_config.get(&target).and_then(|t| t.qemu_rootfs.as_ref()).map(|p| &**p) - } - - /// Path to the python interpreter to use - fn python(&self) -> &Path { - if self.config.build.ends_with("apple-darwin") { - // Force /usr/bin/python3 on macOS for LLDB tests because we're loading the - // LLDB plugin's compiled module which only works with the system python - // (namely not Homebrew-installed python) - Path::new("/usr/bin/python3") - } else { - self.config - .python - .as_ref() - .expect("python is required for running LLDB or rustdoc tests") - } - } - - /// Temporary directory that extended error information is emitted to. - fn extended_error_dir(&self) -> PathBuf { - self.out.join("tmp/extended-error-metadata") - } - - /// Tests whether the `compiler` compiling for `target` should be forced to - /// use a stage1 compiler instead. - /// - /// Currently, by default, the build system does not perform a "full - /// bootstrap" by default where we compile the compiler three times. - /// Instead, we compile the compiler two times. The final stage (stage2) - /// just copies the libraries from the previous stage, which is what this - /// method detects. - /// - /// Here we return `true` if: - /// - /// * The build isn't performing a full bootstrap - /// * The `compiler` is in the final stage, 2 - /// * We're not cross-compiling, so the artifacts are already available in - /// stage1 - /// - /// When all of these conditions are met the build will lift artifacts from - /// the previous stage forward. - fn force_use_stage1(&self, stage: u32, target: TargetSelection) -> bool { - !self.config.full_bootstrap - && !self.config.download_rustc() - && stage >= 2 - && (self.hosts.iter().any(|h| *h == target) || target == self.build) - } - - /// Checks whether the `compiler` compiling for `target` should be forced to - /// use a stage2 compiler instead. - /// - /// When we download the pre-compiled version of rustc and compiler stage is >= 2, - /// it should be forced to use a stage2 compiler. - fn force_use_stage2(&self, stage: u32) -> bool { - self.config.download_rustc() && stage >= 2 - } - - /// Given `num` in the form "a.b.c" return a "release string" which - /// describes the release version number. - /// - /// For example on nightly this returns "a.b.c-nightly", on beta it returns - /// "a.b.c-beta.1" and on stable it just returns "a.b.c". - fn release(&self, num: &str) -> String { - match &self.config.channel[..] { - "stable" => num.to_string(), - "beta" => { - if !self.config.omit_git_hash { - format!("{}-beta.{}", num, self.beta_prerelease_version()) - } else { - format!("{num}-beta") - } - } - "nightly" => format!("{num}-nightly"), - _ => format!("{num}-dev"), - } - } - - fn beta_prerelease_version(&self) -> u32 { - fn extract_beta_rev_from_file>(version_file: P) -> Option { - let version = fs::read_to_string(version_file).ok()?; - - extract_beta_rev(&version) - } - - if let Some(s) = self.prerelease_version.get() { - return s; - } - - // First check if there is a version file available. - // If available, we read the beta revision from that file. - // This only happens when building from a source tarball when Git should not be used. - let count = extract_beta_rev_from_file(self.src.join("version")).unwrap_or_else(|| { - // Figure out how many merge commits happened since we branched off master. - // That's our beta number! - // (Note that we use a `..` range, not the `...` symmetric difference.) - output(self.config.git().arg("rev-list").arg("--count").arg("--merges").arg(format!( - "refs/remotes/origin/{}..HEAD", - self.config.stage0_metadata.config.nightly_branch - ))) - }); - let n = count.trim().parse().unwrap(); - self.prerelease_version.set(Some(n)); - n - } - - /// Returns the value of `release` above for Rust itself. - fn rust_release(&self) -> String { - self.release(&self.version) - } - - /// Returns the "package version" for a component given the `num` release - /// number. - /// - /// The package version is typically what shows up in the names of tarballs. - /// For channels like beta/nightly it's just the channel name, otherwise - /// it's the `num` provided. - fn package_vers(&self, num: &str) -> String { - match &self.config.channel[..] { - "stable" => num.to_string(), - "beta" => "beta".to_string(), - "nightly" => "nightly".to_string(), - _ => format!("{num}-dev"), - } - } - - /// Returns the value of `package_vers` above for Rust itself. - fn rust_package_vers(&self) -> String { - self.package_vers(&self.version) - } - - /// Returns the `version` string associated with this compiler for Rust - /// itself. - /// - /// Note that this is a descriptive string which includes the commit date, - /// sha, version, etc. - fn rust_version(&self) -> String { - let mut version = self.rust_info().version(self, &self.version); - if let Some(ref s) = self.config.description { - version.push_str(" ("); - version.push_str(s); - version.push(')'); - } - version - } - - /// Returns the full commit hash. - fn rust_sha(&self) -> Option<&str> { - self.rust_info().sha() - } - - /// Returns the `a.b.c` version that the given package is at. - fn release_num(&self, package: &str) -> String { - let toml_file_name = self.src.join(&format!("src/tools/{package}/Cargo.toml")); - let toml = t!(fs::read_to_string(&toml_file_name)); - for line in toml.lines() { - if let Some(stripped) = - line.strip_prefix("version = \"").and_then(|s| s.strip_suffix("\"")) - { - return stripped.to_owned(); - } - } - - panic!("failed to find version in {package}'s Cargo.toml") - } - - /// Returns `true` if unstable features should be enabled for the compiler - /// we're building. - fn unstable_features(&self) -> bool { - match &self.config.channel[..] { - "stable" | "beta" => false, - "nightly" | _ => true, - } - } - - /// Returns a Vec of all the dependencies of the given root crate, - /// including transitive dependencies and the root itself. Only includes - /// "local" crates (those in the local source tree, not from a registry). - fn in_tree_crates(&self, root: &str, target: Option) -> Vec<&Crate> { - let mut ret = Vec::new(); - let mut list = vec![INTERNER.intern_str(root)]; - let mut visited = HashSet::new(); - while let Some(krate) = list.pop() { - let krate = self - .crates - .get(&krate) - .unwrap_or_else(|| panic!("metadata missing for {krate}: {:?}", self.crates)); - ret.push(krate); - for dep in &krate.deps { - if !self.crates.contains_key(dep) { - // Ignore non-workspace members. - continue; - } - // Don't include optional deps if their features are not - // enabled. Ideally this would be computed from `cargo - // metadata --features …`, but that is somewhat slow. In - // the future, we may want to consider just filtering all - // build and dev dependencies in metadata::build. - if visited.insert(dep) - && (dep != "profiler_builtins" - || target - .map(|t| self.config.profiler_enabled(t)) - .unwrap_or_else(|| self.config.any_profiler_enabled())) - && (dep != "rustc_codegen_llvm" || self.config.llvm_enabled()) - { - list.push(*dep); - } - } - } - ret.sort_unstable_by_key(|krate| krate.name); // reproducible order needed for tests - ret - } - - fn read_stamp_file(&self, stamp: &Path) -> Vec<(PathBuf, DependencyType)> { - if self.config.dry_run() { - return Vec::new(); - } - - if !stamp.exists() { - eprintln!( - "Error: Unable to find the stamp file {}, did you try to keep a nonexistent build stage?", - stamp.display() - ); - crate::exit!(1); - } - - let mut paths = Vec::new(); - let contents = t!(fs::read(stamp), &stamp); - // This is the method we use for extracting paths from the stamp file passed to us. See - // run_cargo for more information (in compile.rs). - for part in contents.split(|b| *b == 0) { - if part.is_empty() { - continue; - } - let dependency_type = match part[0] as char { - 'h' => DependencyType::Host, - 's' => DependencyType::TargetSelfContained, - 't' => DependencyType::Target, - _ => unreachable!(), - }; - let path = PathBuf::from(t!(str::from_utf8(&part[1..]))); - paths.push((path, dependency_type)); - } - paths - } - - /// Copies a file from `src` to `dst` - pub fn copy(&self, src: &Path, dst: &Path) { - self.copy_internal(src, dst, false); - } - - fn copy_internal(&self, src: &Path, dst: &Path, dereference_symlinks: bool) { - if self.config.dry_run() { - return; - } - self.verbose_than(1, &format!("Copy {src:?} to {dst:?}")); - if src == dst { - return; - } - let _ = fs::remove_file(&dst); - let metadata = t!(src.symlink_metadata()); - let mut src = src.to_path_buf(); - if metadata.file_type().is_symlink() { - if dereference_symlinks { - src = t!(fs::canonicalize(src)); - } else { - let link = t!(fs::read_link(src)); - t!(self.symlink_file(link, dst)); - return; - } - } - if let Ok(()) = fs::hard_link(&src, dst) { - // Attempt to "easy copy" by creating a hard link - // (symlinks don't work on windows), but if that fails - // just fall back to a slow `copy` operation. - } else { - if let Err(e) = fs::copy(&src, dst) { - panic!("failed to copy `{}` to `{}`: {}", src.display(), dst.display(), e) - } - t!(fs::set_permissions(dst, metadata.permissions())); - let atime = FileTime::from_last_access_time(&metadata); - let mtime = FileTime::from_last_modification_time(&metadata); - t!(filetime::set_file_times(dst, atime, mtime)); - } - } - - /// Copies the `src` directory recursively to `dst`. Both are assumed to exist - /// when this function is called. - pub fn cp_r(&self, src: &Path, dst: &Path) { - if self.config.dry_run() { - return; - } - for f in self.read_dir(src) { - let path = f.path(); - let name = path.file_name().unwrap(); - let dst = dst.join(name); - if t!(f.file_type()).is_dir() { - t!(fs::create_dir_all(&dst)); - self.cp_r(&path, &dst); - } else { - let _ = fs::remove_file(&dst); - self.copy(&path, &dst); - } - } - } - - /// Copies the `src` directory recursively to `dst`. Both are assumed to exist - /// when this function is called. Unwanted files or directories can be skipped - /// by returning `false` from the filter function. - pub fn cp_filtered(&self, src: &Path, dst: &Path, filter: &dyn Fn(&Path) -> bool) { - // Immediately recurse with an empty relative path - self.recurse_(src, dst, Path::new(""), filter) - } - - // Inner function does the actual work - fn recurse_(&self, src: &Path, dst: &Path, relative: &Path, filter: &dyn Fn(&Path) -> bool) { - for f in self.read_dir(src) { - let path = f.path(); - let name = path.file_name().unwrap(); - let dst = dst.join(name); - let relative = relative.join(name); - // Only copy file or directory if the filter function returns true - if filter(&relative) { - if t!(f.file_type()).is_dir() { - let _ = fs::remove_dir_all(&dst); - self.create_dir(&dst); - self.recurse_(&path, &dst, &relative, filter); - } else { - let _ = fs::remove_file(&dst); - self.copy(&path, &dst); - } - } - } - } - - fn copy_to_folder(&self, src: &Path, dest_folder: &Path) { - let file_name = src.file_name().unwrap(); - let dest = dest_folder.join(file_name); - self.copy(src, &dest); - } - - fn install(&self, src: &Path, dstdir: &Path, perms: u32) { - if self.config.dry_run() { - return; - } - let dst = dstdir.join(src.file_name().unwrap()); - self.verbose_than(1, &format!("Install {src:?} to {dst:?}")); - t!(fs::create_dir_all(dstdir)); - if !src.exists() { - panic!("Error: File \"{}\" not found!", src.display()); - } - self.copy_internal(src, &dst, true); - chmod(&dst, perms); - } - - fn read(&self, path: &Path) -> String { - if self.config.dry_run() { - return String::new(); - } - t!(fs::read_to_string(path)) - } - - fn create_dir(&self, dir: &Path) { - if self.config.dry_run() { - return; - } - t!(fs::create_dir_all(dir)) - } - - fn remove_dir(&self, dir: &Path) { - if self.config.dry_run() { - return; - } - t!(fs::remove_dir_all(dir)) - } - - fn read_dir(&self, dir: &Path) -> impl Iterator { - let iter = match fs::read_dir(dir) { - Ok(v) => v, - Err(_) if self.config.dry_run() => return vec![].into_iter(), - Err(err) => panic!("could not read dir {dir:?}: {err:?}"), - }; - iter.map(|e| t!(e)).collect::>().into_iter() - } - - fn symlink_file, Q: AsRef>(&self, src: P, link: Q) -> io::Result<()> { - #[cfg(unix)] - use std::os::unix::fs::symlink as symlink_file; - #[cfg(windows)] - use std::os::windows::fs::symlink_file; - if !self.config.dry_run() { symlink_file(src.as_ref(), link.as_ref()) } else { Ok(()) } - } - - /// Returns if config.ninja is enabled, and checks for ninja existence, - /// exiting with a nicer error message if not. - fn ninja(&self) -> bool { - let mut cmd_finder = crate::sanity::Finder::new(); - - if self.config.ninja_in_file { - // Some Linux distros rename `ninja` to `ninja-build`. - // CMake can work with either binary name. - if cmd_finder.maybe_have("ninja-build").is_none() - && cmd_finder.maybe_have("ninja").is_none() - { - eprintln!( - " -Couldn't find required command: ninja (or ninja-build) - -You should install ninja as described at -, -or set `ninja = false` in the `[llvm]` section of `config.toml`. -Alternatively, set `download-ci-llvm = true` in that `[llvm]` section -to download LLVM rather than building it. -" - ); - exit!(1); - } - } - - // If ninja isn't enabled but we're building for MSVC then we try - // doubly hard to enable it. It was realized in #43767 that the msbuild - // CMake generator for MSVC doesn't respect configuration options like - // disabling LLVM assertions, which can often be quite important! - // - // In these cases we automatically enable Ninja if we find it in the - // environment. - if !self.config.ninja_in_file - && self.config.build.contains("msvc") - && cmd_finder.maybe_have("ninja").is_some() - { - return true; - } - - self.config.ninja_in_file - } - - pub fn colored_stdout R>(&self, f: F) -> R { - self.colored_stream_inner(StandardStream::stdout, self.config.stdout_is_tty, f) - } - - pub fn colored_stderr R>(&self, f: F) -> R { - self.colored_stream_inner(StandardStream::stderr, self.config.stderr_is_tty, f) - } - - fn colored_stream_inner(&self, constructor: C, is_tty: bool, f: F) -> R - where - C: Fn(ColorChoice) -> StandardStream, - F: FnOnce(&mut dyn WriteColor) -> R, - { - let choice = match self.config.color { - flags::Color::Always => ColorChoice::Always, - flags::Color::Never => ColorChoice::Never, - flags::Color::Auto if !is_tty => ColorChoice::Never, - flags::Color::Auto => ColorChoice::Auto, - }; - let mut stream = constructor(choice); - let result = f(&mut stream); - stream.reset().unwrap(); - result - } -} - -/// Extract the beta revision from the full version string. -/// -/// The full version string looks like "a.b.c-beta.y". And we need to extract -/// the "y" part from the string. -pub fn extract_beta_rev(version: &str) -> Option { - let parts = version.splitn(2, "-beta.").collect::>(); - let count = parts.get(1).and_then(|s| s.find(' ').map(|p| (&s[..p]).to_string())); - - count -} - -#[cfg(unix)] -fn chmod(path: &Path, perms: u32) { - use std::os::unix::fs::*; - t!(fs::set_permissions(path, fs::Permissions::from_mode(perms))); -} -#[cfg(windows)] -fn chmod(_path: &Path, _perms: u32) {} - -impl Compiler { - pub fn with_stage(mut self, stage: u32) -> Compiler { - self.stage = stage; - self - } - - /// Returns `true` if this is a snapshot compiler for `build`'s configuration - pub fn is_snapshot(&self, build: &Build) -> bool { - self.stage == 0 && self.host == build.build - } - - /// Returns if this compiler should be treated as a final stage one in the - /// current build session. - /// This takes into account whether we're performing a full bootstrap or - /// not; don't directly compare the stage with `2`! - pub fn is_final_stage(&self, build: &Build) -> bool { - let final_stage = if build.config.full_bootstrap { 2 } else { 1 }; - self.stage >= final_stage - } -} - -fn envify(s: &str) -> String { - s.chars() - .map(|c| match c { - '-' => '_', - c => c, - }) - .flat_map(|c| c.to_uppercase()) - .collect() -} diff --git a/src/bootstrap/llvm.rs b/src/bootstrap/llvm.rs deleted file mode 100644 index 455683158..000000000 --- a/src/bootstrap/llvm.rs +++ /dev/null @@ -1,1357 +0,0 @@ -//! Compilation of native dependencies like LLVM. -//! -//! Native projects like LLVM unfortunately aren't suited just yet for -//! compilation in build scripts that Cargo has. This is because the -//! compilation takes a *very* long time but also because we don't want to -//! compile LLVM 3 times as part of a normal bootstrap (we want it cached). -//! -//! LLVM and compiler-rt are essentially just wired up to everything else to -//! ensure that they're always in place if needed. - -use std::env; -use std::env::consts::EXE_EXTENSION; -use std::ffi::{OsStr, OsString}; -use std::fs::{self, File}; -use std::io; -use std::path::{Path, PathBuf}; -use std::process::Command; - -use crate::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::channel; -use crate::config::{Config, TargetSelection}; -use crate::util::get_clang_cl_resource_dir; -use crate::util::{self, exe, output, t, up_to_date}; -use crate::{CLang, GitRepo, Kind}; - -use build_helper::ci::CiEnv; -use build_helper::git::get_git_merge_base; - -#[derive(Clone)] -pub struct LlvmResult { - /// Path to llvm-config binary. - /// NB: This is always the host llvm-config! - pub llvm_config: PathBuf, - /// Path to LLVM cmake directory for the target. - pub llvm_cmake_dir: PathBuf, -} - -pub struct Meta { - stamp: HashStamp, - res: LlvmResult, - out_dir: PathBuf, - root: String, -} - -// Linker flags to pass to LLVM's CMake invocation. -#[derive(Debug, Clone, Default)] -struct LdFlags { - // CMAKE_EXE_LINKER_FLAGS - exe: OsString, - // CMAKE_SHARED_LINKER_FLAGS - shared: OsString, - // CMAKE_MODULE_LINKER_FLAGS - module: OsString, -} - -impl LdFlags { - fn push_all(&mut self, s: impl AsRef) { - let s = s.as_ref(); - self.exe.push(" "); - self.exe.push(s); - self.shared.push(" "); - self.shared.push(s); - self.module.push(" "); - self.module.push(s); - } -} - -/// This returns whether we've already previously built LLVM. -/// -/// It's used to avoid busting caches during x.py check -- if we've already built -/// LLVM, it's fine for us to not try to avoid doing so. -/// -/// This will return the llvm-config if it can get it (but it will not build it -/// if not). -pub fn prebuilt_llvm_config( - builder: &Builder<'_>, - target: TargetSelection, -) -> Result { - builder.config.maybe_download_ci_llvm(); - - // If we're using a custom LLVM bail out here, but we can only use a - // custom LLVM for the build triple. - if let Some(config) = builder.config.target_config.get(&target) { - if let Some(ref s) = config.llvm_config { - check_llvm_version(builder, s); - let llvm_config = s.to_path_buf(); - let mut llvm_cmake_dir = llvm_config.clone(); - llvm_cmake_dir.pop(); - llvm_cmake_dir.pop(); - llvm_cmake_dir.push("lib"); - llvm_cmake_dir.push("cmake"); - llvm_cmake_dir.push("llvm"); - return Ok(LlvmResult { llvm_config, llvm_cmake_dir }); - } - } - - let root = "src/llvm-project/llvm"; - let out_dir = builder.llvm_out(target); - - let mut llvm_config_ret_dir = builder.llvm_out(builder.config.build); - if !builder.config.build.contains("msvc") || builder.ninja() { - llvm_config_ret_dir.push("build"); - } - llvm_config_ret_dir.push("bin"); - let build_llvm_config = llvm_config_ret_dir.join(exe("llvm-config", builder.config.build)); - let llvm_cmake_dir = out_dir.join("lib/cmake/llvm"); - let res = LlvmResult { llvm_config: build_llvm_config, llvm_cmake_dir }; - - let stamp = out_dir.join("llvm-finished-building"); - let stamp = HashStamp::new(stamp, builder.in_tree_llvm_info.sha()); - - if stamp.is_done() { - if stamp.hash.is_none() { - builder.info( - "Could not determine the LLVM submodule commit hash. \ - Assuming that an LLVM rebuild is not necessary.", - ); - builder.info(&format!( - "To force LLVM to rebuild, remove the file `{}`", - stamp.path.display() - )); - } - return Ok(res); - } - - Err(Meta { stamp, res, out_dir, root: root.into() }) -} - -/// This retrieves the LLVM sha we *want* to use, according to git history. -pub(crate) fn detect_llvm_sha(config: &Config, is_git: bool) -> String { - let llvm_sha = if is_git { - // We proceed in 2 steps. First we get the closest commit that is actually upstream. Then we - // walk back further to the last bors merge commit that actually changed LLVM. The first - // step will fail on CI because only the `auto` branch exists; we just fall back to `HEAD` - // in that case. - let closest_upstream = - get_git_merge_base(Some(&config.src)).unwrap_or_else(|_| "HEAD".into()); - let mut rev_list = config.git(); - rev_list.args(&[ - PathBuf::from("rev-list"), - format!("--author={}", config.stage0_metadata.config.git_merge_commit_email).into(), - "-n1".into(), - "--first-parent".into(), - closest_upstream.into(), - "--".into(), - config.src.join("src/llvm-project"), - config.src.join("src/bootstrap/download-ci-llvm-stamp"), - // the LLVM shared object file is named `LLVM-12-rust-{version}-nightly` - config.src.join("src/version"), - ]); - output(&mut rev_list).trim().to_owned() - } else if let Some(info) = channel::read_commit_info_file(&config.src) { - info.sha.trim().to_owned() - } else { - "".to_owned() - }; - - if llvm_sha.is_empty() { - eprintln!("error: could not find commit hash for downloading LLVM"); - eprintln!("help: maybe your repository history is too shallow?"); - eprintln!("help: consider disabling `download-ci-llvm`"); - eprintln!("help: or fetch enough history to include one upstream commit"); - panic!(); - } - - llvm_sha -} - -/// Returns whether the CI-found LLVM is currently usable. -/// -/// This checks both the build triple platform to confirm we're usable at all, -/// and then verifies if the current HEAD matches the detected LLVM SHA head, -/// in which case LLVM is indicated as not available. -pub(crate) fn is_ci_llvm_available(config: &Config, asserts: bool) -> bool { - // This is currently all tier 1 targets and tier 2 targets with host tools - // (since others may not have CI artifacts) - // https://doc.rust-lang.org/rustc/platform-support.html#tier-1 - let supported_platforms = [ - // tier 1 - ("aarch64-unknown-linux-gnu", false), - ("i686-pc-windows-gnu", false), - ("i686-pc-windows-msvc", false), - ("i686-unknown-linux-gnu", false), - ("x86_64-unknown-linux-gnu", true), - ("x86_64-apple-darwin", true), - ("x86_64-pc-windows-gnu", true), - ("x86_64-pc-windows-msvc", true), - // tier 2 with host tools - ("aarch64-apple-darwin", false), - ("aarch64-pc-windows-msvc", false), - ("aarch64-unknown-linux-musl", false), - ("arm-unknown-linux-gnueabi", false), - ("arm-unknown-linux-gnueabihf", false), - ("armv7-unknown-linux-gnueabihf", false), - ("loongarch64-unknown-linux-gnu", false), - ("mips-unknown-linux-gnu", false), - ("mips64-unknown-linux-gnuabi64", false), - ("mips64el-unknown-linux-gnuabi64", false), - ("mipsel-unknown-linux-gnu", false), - ("powerpc-unknown-linux-gnu", false), - ("powerpc64-unknown-linux-gnu", false), - ("powerpc64le-unknown-linux-gnu", false), - ("riscv64gc-unknown-linux-gnu", false), - ("s390x-unknown-linux-gnu", false), - ("x86_64-unknown-freebsd", false), - ("x86_64-unknown-illumos", false), - ("x86_64-unknown-linux-musl", false), - ("x86_64-unknown-netbsd", false), - ]; - - if !supported_platforms.contains(&(&*config.build.triple, asserts)) - && (asserts || !supported_platforms.contains(&(&*config.build.triple, true))) - { - return false; - } - - if is_ci_llvm_modified(config) { - eprintln!("Detected LLVM as non-available: running in CI and modified LLVM in this change"); - return false; - } - - true -} - -/// Returns true if we're running in CI with modified LLVM (and thus can't download it) -pub(crate) fn is_ci_llvm_modified(config: &Config) -> bool { - CiEnv::is_ci() && config.rust_info.is_managed_git_subrepository() && { - // We assume we have access to git, so it's okay to unconditionally pass - // `true` here. - let llvm_sha = detect_llvm_sha(config, true); - let head_sha = output(config.git().arg("rev-parse").arg("HEAD")); - let head_sha = head_sha.trim(); - llvm_sha == head_sha - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Llvm { - pub target: TargetSelection, -} - -impl Step for Llvm { - type Output = LlvmResult; - - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/llvm-project").path("src/llvm-project/llvm") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Llvm { target: run.target }); - } - - /// Compile LLVM for `target`. - fn run(self, builder: &Builder<'_>) -> LlvmResult { - let target = self.target; - let target_native = if self.target.starts_with("riscv") { - // RISC-V target triples in Rust is not named the same as C compiler target triples. - // This converts Rust RISC-V target triples to C compiler triples. - let idx = target.triple.find('-').unwrap(); - - format!("riscv{}{}", &target.triple[5..7], &target.triple[idx..]) - } else if self.target.starts_with("powerpc") && self.target.ends_with("freebsd") { - // FreeBSD 13 had incompatible ABI changes on all PowerPC platforms. - // Set the version suffix to 13.0 so the correct target details are used. - format!("{}{}", self.target, "13.0") - } else { - target.to_string() - }; - - let Meta { stamp, res, out_dir, root } = match prebuilt_llvm_config(builder, target) { - Ok(p) => return p, - Err(m) => m, - }; - - builder.update_submodule(&Path::new("src").join("llvm-project")); - if builder.llvm_link_shared() && target.contains("windows") { - panic!("shared linking to LLVM is not currently supported on {}", target.triple); - } - - let _guard = builder.msg_unstaged(Kind::Build, "LLVM", target); - t!(stamp.remove()); - let _time = util::timeit(&builder); - t!(fs::create_dir_all(&out_dir)); - - // https://llvm.org/docs/CMake.html - let mut cfg = cmake::Config::new(builder.src.join(root)); - let mut ldflags = LdFlags::default(); - - let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) { - (false, _) => "Debug", - (true, false) => "Release", - (true, true) => "RelWithDebInfo", - }; - - // NOTE: remember to also update `config.example.toml` when changing the - // defaults! - let llvm_targets = match &builder.config.llvm_targets { - Some(s) => s, - None => { - "AArch64;ARM;BPF;Hexagon;LoongArch;MSP430;Mips;NVPTX;PowerPC;RISCV;\ - Sparc;SystemZ;WebAssembly;X86" - } - }; - - let llvm_exp_targets = match builder.config.llvm_experimental_targets { - Some(ref s) => s, - None => "AVR;M68k;CSKY", - }; - - let assertions = if builder.config.llvm_assertions { "ON" } else { "OFF" }; - let plugins = if builder.config.llvm_plugins { "ON" } else { "OFF" }; - let enable_tests = if builder.config.llvm_tests { "ON" } else { "OFF" }; - let enable_warnings = if builder.config.llvm_enable_warnings { "ON" } else { "OFF" }; - - cfg.out_dir(&out_dir) - .profile(profile) - .define("LLVM_ENABLE_ASSERTIONS", assertions) - .define("LLVM_UNREACHABLE_OPTIMIZE", "OFF") - .define("LLVM_ENABLE_PLUGINS", plugins) - .define("LLVM_TARGETS_TO_BUILD", llvm_targets) - .define("LLVM_EXPERIMENTAL_TARGETS_TO_BUILD", llvm_exp_targets) - .define("LLVM_INCLUDE_EXAMPLES", "OFF") - .define("LLVM_INCLUDE_DOCS", "OFF") - .define("LLVM_INCLUDE_BENCHMARKS", "OFF") - .define("LLVM_INCLUDE_TESTS", enable_tests) - .define("LLVM_ENABLE_TERMINFO", "OFF") - .define("LLVM_ENABLE_LIBEDIT", "OFF") - .define("LLVM_ENABLE_BINDINGS", "OFF") - .define("LLVM_ENABLE_Z3_SOLVER", "OFF") - .define("LLVM_PARALLEL_COMPILE_JOBS", builder.jobs().to_string()) - .define("LLVM_TARGET_ARCH", target_native.split('-').next().unwrap()) - .define("LLVM_DEFAULT_TARGET_TRIPLE", target_native) - .define("LLVM_ENABLE_WARNINGS", enable_warnings); - - // Parts of our test suite rely on the `FileCheck` tool, which is built by default in - // `build/$TARGET/llvm/build/bin` is but *not* then installed to `build/$TARGET/llvm/bin`. - // This flag makes sure `FileCheck` is copied in the final binaries directory. - cfg.define("LLVM_INSTALL_UTILS", "ON"); - - if builder.config.llvm_profile_generate { - cfg.define("LLVM_BUILD_INSTRUMENTED", "IR"); - if let Ok(llvm_profile_dir) = std::env::var("LLVM_PROFILE_DIR") { - cfg.define("LLVM_PROFILE_DATA_DIR", llvm_profile_dir); - } - cfg.define("LLVM_BUILD_RUNTIME", "No"); - } - if let Some(path) = builder.config.llvm_profile_use.as_ref() { - cfg.define("LLVM_PROFDATA_FILE", &path); - } - - // Disable zstd to avoid a dependency on libzstd.so. - cfg.define("LLVM_ENABLE_ZSTD", "OFF"); - - if !target.contains("windows") { - cfg.define("LLVM_ENABLE_ZLIB", "ON"); - } else { - cfg.define("LLVM_ENABLE_ZLIB", "OFF"); - } - - // Are we compiling for iOS/tvOS/watchOS? - if target.contains("apple-ios") - || target.contains("apple-tvos") - || target.contains("apple-watchos") - { - // These two defines prevent CMake from automatically trying to add a MacOSX sysroot, which leads to a compiler error. - cfg.define("CMAKE_OSX_SYSROOT", "/"); - cfg.define("CMAKE_OSX_DEPLOYMENT_TARGET", ""); - // Prevent cmake from adding -bundle to CFLAGS automatically, which leads to a compiler error because "-bitcode_bundle" also gets added. - cfg.define("LLVM_ENABLE_PLUGINS", "OFF"); - // Zlib fails to link properly, leading to a compiler error. - cfg.define("LLVM_ENABLE_ZLIB", "OFF"); - } - - // This setting makes the LLVM tools link to the dynamic LLVM library, - // which saves both memory during parallel links and overall disk space - // for the tools. We don't do this on every platform as it doesn't work - // equally well everywhere. - if builder.llvm_link_shared() { - cfg.define("LLVM_LINK_LLVM_DYLIB", "ON"); - } - - if (target.starts_with("riscv") || target.starts_with("csky")) - && !target.contains("freebsd") - && !target.contains("openbsd") - && !target.contains("netbsd") - { - // RISC-V and CSKY GCC erroneously requires linking against - // `libatomic` when using 1-byte and 2-byte C++ - // atomics but the LLVM build system check cannot - // detect this. Therefore it is set manually here. - // Some BSD uses Clang as its system compiler and - // provides no libatomic in its base system so does - // not want this. - ldflags.exe.push(" -latomic"); - ldflags.shared.push(" -latomic"); - } - - if target.contains("msvc") { - cfg.define("LLVM_USE_CRT_DEBUG", "MT"); - cfg.define("LLVM_USE_CRT_RELEASE", "MT"); - cfg.define("LLVM_USE_CRT_RELWITHDEBINFO", "MT"); - cfg.static_crt(true); - } - - if target.starts_with("i686") { - cfg.define("LLVM_BUILD_32_BITS", "ON"); - } - - let mut enabled_llvm_projects = Vec::new(); - - if util::forcing_clang_based_tests() { - enabled_llvm_projects.push("clang"); - enabled_llvm_projects.push("compiler-rt"); - } - - if builder.config.llvm_polly { - enabled_llvm_projects.push("polly"); - } - - if builder.config.llvm_clang { - enabled_llvm_projects.push("clang"); - } - - // We want libxml to be disabled. - // See https://github.com/rust-lang/rust/pull/50104 - cfg.define("LLVM_ENABLE_LIBXML2", "OFF"); - - if !enabled_llvm_projects.is_empty() { - enabled_llvm_projects.sort(); - enabled_llvm_projects.dedup(); - cfg.define("LLVM_ENABLE_PROJECTS", enabled_llvm_projects.join(";")); - } - - if let Some(num_linkers) = builder.config.llvm_link_jobs { - if num_linkers > 0 { - cfg.define("LLVM_PARALLEL_LINK_JOBS", num_linkers.to_string()); - } - } - - // https://llvm.org/docs/HowToCrossCompileLLVM.html - if target != builder.config.build { - let LlvmResult { llvm_config, .. } = - builder.ensure(Llvm { target: builder.config.build }); - if !builder.config.dry_run() { - let llvm_bindir = output(Command::new(&llvm_config).arg("--bindir")); - let host_bin = Path::new(llvm_bindir.trim()); - cfg.define( - "LLVM_TABLEGEN", - host_bin.join("llvm-tblgen").with_extension(EXE_EXTENSION), - ); - // LLVM_NM is required for cross compiling using MSVC - cfg.define("LLVM_NM", host_bin.join("llvm-nm").with_extension(EXE_EXTENSION)); - } - cfg.define("LLVM_CONFIG_PATH", llvm_config); - if builder.config.llvm_clang { - let build_bin = builder.llvm_out(builder.config.build).join("build").join("bin"); - let clang_tblgen = build_bin.join("clang-tblgen").with_extension(EXE_EXTENSION); - if !builder.config.dry_run() && !clang_tblgen.exists() { - panic!("unable to find {}", clang_tblgen.display()); - } - cfg.define("CLANG_TABLEGEN", clang_tblgen); - } - } - - let llvm_version_suffix = if let Some(ref suffix) = builder.config.llvm_version_suffix { - // Allow version-suffix="" to not define a version suffix at all. - if !suffix.is_empty() { Some(suffix.to_string()) } else { None } - } else if builder.config.channel == "dev" { - // Changes to a version suffix require a complete rebuild of the LLVM. - // To avoid rebuilds during a time of version bump, don't include rustc - // release number on the dev channel. - Some("-rust-dev".to_string()) - } else { - Some(format!("-rust-{}-{}", builder.version, builder.config.channel)) - }; - if let Some(ref suffix) = llvm_version_suffix { - cfg.define("LLVM_VERSION_SUFFIX", suffix); - } - - configure_cmake(builder, target, &mut cfg, true, ldflags, &[]); - configure_llvm(builder, target, &mut cfg); - - for (key, val) in &builder.config.llvm_build_config { - cfg.define(key, val); - } - - if builder.config.dry_run() { - return res; - } - - cfg.build(); - - // Helper to find the name of LLVM's shared library on darwin and linux. - let find_llvm_lib_name = |extension| { - let mut cmd = Command::new(&res.llvm_config); - let version = output(cmd.arg("--version")); - let major = version.split('.').next().unwrap(); - - match &llvm_version_suffix { - Some(version_suffix) => format!("libLLVM-{major}{version_suffix}.{extension}"), - None => format!("libLLVM-{major}.{extension}"), - } - }; - - // When building LLVM with LLVM_LINK_LLVM_DYLIB for macOS, an unversioned - // libLLVM.dylib will be built. However, llvm-config will still look - // for a versioned path like libLLVM-14.dylib. Manually create a symbolic - // link to make llvm-config happy. - if builder.llvm_link_shared() && target.contains("apple-darwin") { - let lib_name = find_llvm_lib_name("dylib"); - let lib_llvm = out_dir.join("build").join("lib").join(lib_name); - if !lib_llvm.exists() { - t!(builder.symlink_file("libLLVM.dylib", &lib_llvm)); - } - } - - // When building LLVM as a shared library on linux, it can contain unexpected debuginfo: - // some can come from the C++ standard library. Unless we're explicitly requesting LLVM to - // be built with debuginfo, strip it away after the fact, to make dist artifacts smaller. - if builder.llvm_link_shared() - && builder.config.llvm_optimize - && !builder.config.llvm_release_debuginfo - { - // Find the name of the LLVM shared library that we just built. - let lib_name = find_llvm_lib_name("so"); - - // If the shared library exists in LLVM's `/build/lib/` or `/lib/` folders, strip its - // debuginfo. - crate::compile::strip_debug(builder, target, &out_dir.join("lib").join(&lib_name)); - crate::compile::strip_debug( - builder, - target, - &out_dir.join("build").join("lib").join(&lib_name), - ); - } - - t!(stamp.write()); - - res - } -} - -fn check_llvm_version(builder: &Builder<'_>, llvm_config: &Path) { - if builder.config.dry_run() { - return; - } - - let mut cmd = Command::new(llvm_config); - let version = output(cmd.arg("--version")); - let mut parts = version.split('.').take(2).filter_map(|s| s.parse::().ok()); - if let (Some(major), Some(_minor)) = (parts.next(), parts.next()) { - if major >= 15 { - return; - } - } - panic!("\n\nbad LLVM version: {version}, need >=15.0\n\n") -} - -fn configure_cmake( - builder: &Builder<'_>, - target: TargetSelection, - cfg: &mut cmake::Config, - use_compiler_launcher: bool, - mut ldflags: LdFlags, - extra_compiler_flags: &[&str], -) { - // Do not print installation messages for up-to-date files. - // LLVM and LLD builds can produce a lot of those and hit CI limits on log size. - cfg.define("CMAKE_INSTALL_MESSAGE", "LAZY"); - - // Do not allow the user's value of DESTDIR to influence where - // LLVM will install itself. LLVM must always be installed in our - // own build directories. - cfg.env("DESTDIR", ""); - - if builder.ninja() { - cfg.generator("Ninja"); - } - cfg.target(&target.triple).host(&builder.config.build.triple); - - if target != builder.config.build { - cfg.define("CMAKE_CROSSCOMPILING", "True"); - - if target.contains("netbsd") { - cfg.define("CMAKE_SYSTEM_NAME", "NetBSD"); - } else if target.contains("dragonfly") { - cfg.define("CMAKE_SYSTEM_NAME", "DragonFly"); - } else if target.contains("freebsd") { - cfg.define("CMAKE_SYSTEM_NAME", "FreeBSD"); - } else if target.contains("windows") { - cfg.define("CMAKE_SYSTEM_NAME", "Windows"); - } else if target.contains("haiku") { - cfg.define("CMAKE_SYSTEM_NAME", "Haiku"); - } else if target.contains("solaris") || target.contains("illumos") { - cfg.define("CMAKE_SYSTEM_NAME", "SunOS"); - } else if target.contains("linux") { - cfg.define("CMAKE_SYSTEM_NAME", "Linux"); - } else { - builder.info(&format!( - "could not determine CMAKE_SYSTEM_NAME from the target `{target}`, build may fail", - )); - } - - // When cross-compiling we should also set CMAKE_SYSTEM_VERSION, but in - // that case like CMake we cannot easily determine system version either. - // - // Since, the LLVM itself makes rather limited use of version checks in - // CMakeFiles (and then only in tests), and so far no issues have been - // reported, the system version is currently left unset. - - if target.contains("darwin") { - // Make sure that CMake does not build universal binaries on macOS. - // Explicitly specify the one single target architecture. - if target.starts_with("aarch64") { - // macOS uses a different name for building arm64 - cfg.define("CMAKE_OSX_ARCHITECTURES", "arm64"); - } else if target.starts_with("i686") { - // macOS uses a different name for building i386 - cfg.define("CMAKE_OSX_ARCHITECTURES", "i386"); - } else { - cfg.define("CMAKE_OSX_ARCHITECTURES", target.triple.split('-').next().unwrap()); - } - } - } - - let sanitize_cc = |cc: &Path| { - if target.contains("msvc") { - OsString::from(cc.to_str().unwrap().replace("\\", "/")) - } else { - cc.as_os_str().to_owned() - } - }; - - // MSVC with CMake uses msbuild by default which doesn't respect these - // vars that we'd otherwise configure. In that case we just skip this - // entirely. - if target.contains("msvc") && !builder.ninja() { - return; - } - - let (cc, cxx) = match builder.config.llvm_clang_cl { - Some(ref cl) => (cl.into(), cl.into()), - None => (builder.cc(target), builder.cxx(target).unwrap()), - }; - - // Handle msvc + ninja + ccache specially (this is what the bots use) - if target.contains("msvc") && builder.ninja() && builder.config.ccache.is_some() { - let mut wrap_cc = env::current_exe().expect("failed to get cwd"); - wrap_cc.set_file_name("sccache-plus-cl.exe"); - - cfg.define("CMAKE_C_COMPILER", sanitize_cc(&wrap_cc)) - .define("CMAKE_CXX_COMPILER", sanitize_cc(&wrap_cc)); - cfg.env("SCCACHE_PATH", builder.config.ccache.as_ref().unwrap()) - .env("SCCACHE_TARGET", target.triple) - .env("SCCACHE_CC", &cc) - .env("SCCACHE_CXX", &cxx); - - // Building LLVM on MSVC can be a little ludicrous at times. We're so far - // off the beaten path here that I'm not really sure this is even half - // supported any more. Here we're trying to: - // - // * Build LLVM on MSVC - // * Build LLVM with `clang-cl` instead of `cl.exe` - // * Build a project with `sccache` - // * Build for 32-bit as well - // * Build with Ninja - // - // For `cl.exe` there are different binaries to compile 32/64 bit which - // we use but for `clang-cl` there's only one which internally - // multiplexes via flags. As a result it appears that CMake's detection - // of a compiler's architecture and such on MSVC **doesn't** pass any - // custom flags we pass in CMAKE_CXX_FLAGS below. This means that if we - // use `clang-cl.exe` it's always diagnosed as a 64-bit compiler which - // definitely causes problems since all the env vars are pointing to - // 32-bit libraries. - // - // To hack around this... again... we pass an argument that's - // unconditionally passed in the sccache shim. This'll get CMake to - // correctly diagnose it's doing a 32-bit compilation and LLVM will - // internally configure itself appropriately. - if builder.config.llvm_clang_cl.is_some() && target.contains("i686") { - cfg.env("SCCACHE_EXTRA_ARGS", "-m32"); - } - } else { - // If ccache is configured we inform the build a little differently how - // to invoke ccache while also invoking our compilers. - if use_compiler_launcher { - if let Some(ref ccache) = builder.config.ccache { - cfg.define("CMAKE_C_COMPILER_LAUNCHER", ccache) - .define("CMAKE_CXX_COMPILER_LAUNCHER", ccache); - } - } - cfg.define("CMAKE_C_COMPILER", sanitize_cc(&cc)) - .define("CMAKE_CXX_COMPILER", sanitize_cc(&cxx)) - .define("CMAKE_ASM_COMPILER", sanitize_cc(&cc)); - } - - cfg.build_arg("-j").build_arg(builder.jobs().to_string()); - let mut cflags: OsString = builder.cflags(target, GitRepo::Llvm, CLang::C).join(" ").into(); - if let Some(ref s) = builder.config.llvm_cflags { - cflags.push(" "); - cflags.push(s); - } - - if builder.config.llvm_clang_cl.is_some() { - cflags.push(&format!(" --target={target}")); - } - for flag in extra_compiler_flags { - cflags.push(&format!(" {flag}")); - } - cfg.define("CMAKE_C_FLAGS", cflags); - let mut cxxflags: OsString = builder.cflags(target, GitRepo::Llvm, CLang::Cxx).join(" ").into(); - if let Some(ref s) = builder.config.llvm_cxxflags { - cxxflags.push(" "); - cxxflags.push(s); - } - if builder.config.llvm_clang_cl.is_some() { - cxxflags.push(&format!(" --target={target}")); - } - for flag in extra_compiler_flags { - cxxflags.push(&format!(" {flag}")); - } - cfg.define("CMAKE_CXX_FLAGS", cxxflags); - if let Some(ar) = builder.ar(target) { - if ar.is_absolute() { - // LLVM build breaks if `CMAKE_AR` is a relative path, for some reason it - // tries to resolve this path in the LLVM build directory. - cfg.define("CMAKE_AR", sanitize_cc(&ar)); - } - } - - if let Some(ranlib) = builder.ranlib(target) { - if ranlib.is_absolute() { - // LLVM build breaks if `CMAKE_RANLIB` is a relative path, for some reason it - // tries to resolve this path in the LLVM build directory. - cfg.define("CMAKE_RANLIB", sanitize_cc(&ranlib)); - } - } - - if let Some(ref flags) = builder.config.llvm_ldflags { - ldflags.push_all(flags); - } - - if let Some(flags) = get_var("LDFLAGS", &builder.config.build.triple, &target.triple) { - ldflags.push_all(&flags); - } - - // For distribution we want the LLVM tools to be *statically* linked to libstdc++. - // We also do this if the user explicitly requested static libstdc++. - if builder.config.llvm_static_stdcpp - && !target.contains("msvc") - && !target.contains("netbsd") - && !target.contains("solaris") - { - if target.contains("apple") || target.contains("windows") { - ldflags.push_all("-static-libstdc++"); - } else { - ldflags.push_all("-Wl,-Bsymbolic -static-libstdc++"); - } - } - - cfg.define("CMAKE_SHARED_LINKER_FLAGS", &ldflags.shared); - cfg.define("CMAKE_MODULE_LINKER_FLAGS", &ldflags.module); - cfg.define("CMAKE_EXE_LINKER_FLAGS", &ldflags.exe); - - if env::var_os("SCCACHE_ERROR_LOG").is_some() { - cfg.env("RUSTC_LOG", "sccache=warn"); - } -} - -fn configure_llvm(builder: &Builder<'_>, target: TargetSelection, cfg: &mut cmake::Config) { - // ThinLTO is only available when building with LLVM, enabling LLD is required. - // Apple's linker ld64 supports ThinLTO out of the box though, so don't use LLD on Darwin. - if builder.config.llvm_thin_lto { - cfg.define("LLVM_ENABLE_LTO", "Thin"); - if !target.contains("apple") { - cfg.define("LLVM_ENABLE_LLD", "ON"); - } - } - - if let Some(ref linker) = builder.config.llvm_use_linker { - cfg.define("LLVM_USE_LINKER", linker); - } - - if builder.config.llvm_allow_old_toolchain { - cfg.define("LLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN", "YES"); - } -} - -// Adapted from https://github.com/alexcrichton/cc-rs/blob/fba7feded71ee4f63cfe885673ead6d7b4f2f454/src/lib.rs#L2347-L2365 -fn get_var(var_base: &str, host: &str, target: &str) -> Option { - let kind = if host == target { "HOST" } else { "TARGET" }; - let target_u = target.replace("-", "_"); - env::var_os(&format!("{var_base}_{target}")) - .or_else(|| env::var_os(&format!("{}_{}", var_base, target_u))) - .or_else(|| env::var_os(&format!("{}_{}", kind, var_base))) - .or_else(|| env::var_os(var_base)) -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Lld { - pub target: TargetSelection, -} - -impl Step for Lld { - type Output = PathBuf; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/llvm-project/lld") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Lld { target: run.target }); - } - - /// Compile LLD for `target`. - fn run(self, builder: &Builder<'_>) -> PathBuf { - if builder.config.dry_run() { - return PathBuf::from("lld-out-dir-test-gen"); - } - let target = self.target; - - let LlvmResult { llvm_config, llvm_cmake_dir } = builder.ensure(Llvm { target }); - - // The `dist` step packages LLD next to LLVM's binaries for download-ci-llvm. The root path - // we usually expect here is `./build/$triple/ci-llvm/`, with the binaries in its `bin` - // subfolder. We check if that's the case, and if LLD's binary already exists there next to - // `llvm-config`: if so, we can use it instead of building LLVM/LLD from source. - let ci_llvm_bin = llvm_config.parent().unwrap(); - if ci_llvm_bin.is_dir() && ci_llvm_bin.file_name().unwrap() == "bin" { - let lld_path = ci_llvm_bin.join(exe("lld", target)); - if lld_path.exists() { - // The following steps copying `lld` as `rust-lld` to the sysroot, expect it in the - // `bin` subfolder of this step's out dir. - return ci_llvm_bin.parent().unwrap().to_path_buf(); - } - } - - let out_dir = builder.lld_out(target); - let done_stamp = out_dir.join("lld-finished-building"); - if done_stamp.exists() { - return out_dir; - } - - let _guard = builder.msg_unstaged(Kind::Build, "LLD", target); - let _time = util::timeit(&builder); - t!(fs::create_dir_all(&out_dir)); - - let mut cfg = cmake::Config::new(builder.src.join("src/llvm-project/lld")); - let mut ldflags = LdFlags::default(); - - // When building LLD as part of a build with instrumentation on windows, for example - // when doing PGO on CI, cmake or clang-cl don't automatically link clang's - // profiler runtime in. In that case, we need to manually ask cmake to do it, to avoid - // linking errors, much like LLVM's cmake setup does in that situation. - if builder.config.llvm_profile_generate && target.contains("msvc") { - if let Some(clang_cl_path) = builder.config.llvm_clang_cl.as_ref() { - // Find clang's runtime library directory and push that as a search path to the - // cmake linker flags. - let clang_rt_dir = get_clang_cl_resource_dir(clang_cl_path); - ldflags.push_all(&format!("/libpath:{}", clang_rt_dir.display())); - } - } - - // LLD is built as an LLVM tool, but is distributed outside of the `llvm-tools` component, - // which impacts where it expects to find LLVM's shared library. This causes #80703. - // - // LLD is distributed at "$root/lib/rustlib/$host/bin/rust-lld", but the `libLLVM-*.so` it - // needs is distributed at "$root/lib". The default rpath of "$ORIGIN/../lib" points at the - // lib path for LLVM tools, not the one for rust binaries. - // - // (The `llvm-tools` component copies the .so there for the other tools, and with that - // component installed, one can successfully invoke `rust-lld` directly without rustup's - // `LD_LIBRARY_PATH` overrides) - // - if builder.config.rpath_enabled(target) - && util::use_host_linker(target) - && builder.config.llvm_link_shared() - && target.contains("linux") - { - // So we inform LLD where it can find LLVM's libraries by adding an rpath entry to the - // expected parent `lib` directory. - // - // Be careful when changing this path, we need to ensure it's quoted or escaped: - // `$ORIGIN` would otherwise be expanded when the `LdFlags` are passed verbatim to - // cmake. - ldflags.push_all("-Wl,-rpath,'$ORIGIN/../../../'"); - } - - configure_cmake(builder, target, &mut cfg, true, ldflags, &[]); - configure_llvm(builder, target, &mut cfg); - - // Re-use the same flags as llvm to control the level of debug information - // generated for lld. - let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) { - (false, _) => "Debug", - (true, false) => "Release", - (true, true) => "RelWithDebInfo", - }; - - cfg.out_dir(&out_dir) - .profile(profile) - .define("LLVM_CMAKE_DIR", llvm_cmake_dir) - .define("LLVM_INCLUDE_TESTS", "OFF"); - - if target != builder.config.build { - // Use the host llvm-tblgen binary. - cfg.define( - "LLVM_TABLEGEN_EXE", - llvm_config.with_file_name("llvm-tblgen").with_extension(EXE_EXTENSION), - ); - } - - cfg.build(); - - t!(File::create(&done_stamp)); - out_dir - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Sanitizers { - pub target: TargetSelection, -} - -impl Step for Sanitizers { - type Output = Vec; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("sanitizers") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Sanitizers { target: run.target }); - } - - /// Builds sanitizer runtime libraries. - fn run(self, builder: &Builder<'_>) -> Self::Output { - let compiler_rt_dir = builder.src.join("src/llvm-project/compiler-rt"); - if !compiler_rt_dir.exists() { - return Vec::new(); - } - - let out_dir = builder.native_dir(self.target).join("sanitizers"); - let runtimes = supported_sanitizers(&out_dir, self.target, &builder.config.channel); - if runtimes.is_empty() { - return runtimes; - } - - let LlvmResult { llvm_config, .. } = builder.ensure(Llvm { target: builder.config.build }); - if builder.config.dry_run() { - return runtimes; - } - - let stamp = out_dir.join("sanitizers-finished-building"); - let stamp = HashStamp::new(stamp, builder.in_tree_llvm_info.sha()); - - if stamp.is_done() { - if stamp.hash.is_none() { - builder.info(&format!( - "Rebuild sanitizers by removing the file `{}`", - stamp.path.display() - )); - } - return runtimes; - } - - let _guard = builder.msg_unstaged(Kind::Build, "sanitizers", self.target); - t!(stamp.remove()); - let _time = util::timeit(&builder); - - let mut cfg = cmake::Config::new(&compiler_rt_dir); - cfg.profile("Release"); - cfg.define("CMAKE_C_COMPILER_TARGET", self.target.triple); - cfg.define("COMPILER_RT_BUILD_BUILTINS", "OFF"); - cfg.define("COMPILER_RT_BUILD_CRT", "OFF"); - cfg.define("COMPILER_RT_BUILD_LIBFUZZER", "OFF"); - cfg.define("COMPILER_RT_BUILD_PROFILE", "OFF"); - cfg.define("COMPILER_RT_BUILD_SANITIZERS", "ON"); - cfg.define("COMPILER_RT_BUILD_XRAY", "OFF"); - cfg.define("COMPILER_RT_DEFAULT_TARGET_ONLY", "ON"); - cfg.define("COMPILER_RT_USE_LIBCXX", "OFF"); - cfg.define("LLVM_CONFIG_PATH", &llvm_config); - - // On Darwin targets the sanitizer runtimes are build as universal binaries. - // Unfortunately sccache currently lacks support to build them successfully. - // Disable compiler launcher on Darwin targets to avoid potential issues. - let use_compiler_launcher = !self.target.contains("apple-darwin"); - let extra_compiler_flags: &[&str] = - if self.target.contains("apple") { &["-fembed-bitcode=off"] } else { &[] }; - configure_cmake( - builder, - self.target, - &mut cfg, - use_compiler_launcher, - LdFlags::default(), - extra_compiler_flags, - ); - - t!(fs::create_dir_all(&out_dir)); - cfg.out_dir(out_dir); - - for runtime in &runtimes { - cfg.build_target(&runtime.cmake_target); - cfg.build(); - } - t!(stamp.write()); - - runtimes - } -} - -#[derive(Clone, Debug)] -pub struct SanitizerRuntime { - /// CMake target used to build the runtime. - pub cmake_target: String, - /// Path to the built runtime library. - pub path: PathBuf, - /// Library filename that will be used rustc. - pub name: String, -} - -/// Returns sanitizers available on a given target. -fn supported_sanitizers( - out_dir: &Path, - target: TargetSelection, - channel: &str, -) -> Vec { - let darwin_libs = |os: &str, components: &[&str]| -> Vec { - components - .iter() - .map(move |c| SanitizerRuntime { - cmake_target: format!("clang_rt.{}_{}_dynamic", c, os), - path: out_dir - .join(&format!("build/lib/darwin/libclang_rt.{}_{}_dynamic.dylib", c, os)), - name: format!("librustc-{}_rt.{}.dylib", channel, c), - }) - .collect() - }; - - let common_libs = |os: &str, arch: &str, components: &[&str]| -> Vec { - components - .iter() - .map(move |c| SanitizerRuntime { - cmake_target: format!("clang_rt.{}-{}", c, arch), - path: out_dir.join(&format!("build/lib/{}/libclang_rt.{}-{}.a", os, c, arch)), - name: format!("librustc-{}_rt.{}.a", channel, c), - }) - .collect() - }; - - match &*target.triple { - "aarch64-apple-darwin" => darwin_libs("osx", &["asan", "lsan", "tsan"]), - "aarch64-apple-ios" => darwin_libs("ios", &["asan", "tsan"]), - "aarch64-apple-ios-sim" => darwin_libs("iossim", &["asan", "tsan"]), - "aarch64-apple-ios-macabi" => darwin_libs("osx", &["asan", "lsan", "tsan"]), - "aarch64-unknown-fuchsia" => common_libs("fuchsia", "aarch64", &["asan"]), - "aarch64-unknown-linux-gnu" => { - common_libs("linux", "aarch64", &["asan", "lsan", "msan", "tsan", "hwasan"]) - } - "aarch64-unknown-linux-ohos" => { - common_libs("linux", "aarch64", &["asan", "lsan", "msan", "tsan", "hwasan"]) - } - "x86_64-apple-darwin" => darwin_libs("osx", &["asan", "lsan", "tsan"]), - "x86_64-unknown-fuchsia" => common_libs("fuchsia", "x86_64", &["asan"]), - "x86_64-apple-ios" => darwin_libs("iossim", &["asan", "tsan"]), - "x86_64-apple-ios-macabi" => darwin_libs("osx", &["asan", "lsan", "tsan"]), - "x86_64-unknown-freebsd" => common_libs("freebsd", "x86_64", &["asan", "msan", "tsan"]), - "x86_64-unknown-netbsd" => { - common_libs("netbsd", "x86_64", &["asan", "lsan", "msan", "tsan"]) - } - "x86_64-unknown-illumos" => common_libs("illumos", "x86_64", &["asan"]), - "x86_64-pc-solaris" => common_libs("solaris", "x86_64", &["asan"]), - "x86_64-unknown-linux-gnu" => { - common_libs("linux", "x86_64", &["asan", "lsan", "msan", "safestack", "tsan"]) - } - "x86_64-unknown-linux-musl" => { - common_libs("linux", "x86_64", &["asan", "lsan", "msan", "tsan"]) - } - "s390x-unknown-linux-gnu" => { - common_libs("linux", "s390x", &["asan", "lsan", "msan", "tsan"]) - } - "s390x-unknown-linux-musl" => { - common_libs("linux", "s390x", &["asan", "lsan", "msan", "tsan"]) - } - "x86_64-unknown-linux-ohos" => { - common_libs("linux", "x86_64", &["asan", "lsan", "msan", "tsan"]) - } - _ => Vec::new(), - } -} - -struct HashStamp { - path: PathBuf, - hash: Option>, -} - -impl HashStamp { - fn new(path: PathBuf, hash: Option<&str>) -> Self { - HashStamp { path, hash: hash.map(|s| s.as_bytes().to_owned()) } - } - - fn is_done(&self) -> bool { - match fs::read(&self.path) { - Ok(h) => self.hash.as_deref().unwrap_or(b"") == h.as_slice(), - Err(e) if e.kind() == io::ErrorKind::NotFound => false, - Err(e) => { - panic!("failed to read stamp file `{}`: {}", self.path.display(), e); - } - } - } - - fn remove(&self) -> io::Result<()> { - match fs::remove_file(&self.path) { - Ok(()) => Ok(()), - Err(e) => { - if e.kind() == io::ErrorKind::NotFound { - Ok(()) - } else { - Err(e) - } - } - } - } - - fn write(&self) -> io::Result<()> { - fs::write(&self.path, self.hash.as_deref().unwrap_or(b"")) - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct CrtBeginEnd { - pub target: TargetSelection, -} - -impl Step for CrtBeginEnd { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/llvm-project/compiler-rt/lib/crt") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(CrtBeginEnd { target: run.target }); - } - - /// Build crtbegin.o/crtend.o for musl target. - fn run(self, builder: &Builder<'_>) -> Self::Output { - builder.update_submodule(&Path::new("src/llvm-project")); - - let out_dir = builder.native_dir(self.target).join("crt"); - - if builder.config.dry_run() { - return out_dir; - } - - let crtbegin_src = builder.src.join("src/llvm-project/compiler-rt/lib/builtins/crtbegin.c"); - let crtend_src = builder.src.join("src/llvm-project/compiler-rt/lib/builtins/crtend.c"); - if up_to_date(&crtbegin_src, &out_dir.join("crtbegin.o")) - && up_to_date(&crtend_src, &out_dir.join("crtendS.o")) - { - return out_dir; - } - - let _guard = builder.msg_unstaged(Kind::Build, "crtbegin.o and crtend.o", self.target); - t!(fs::create_dir_all(&out_dir)); - - let mut cfg = cc::Build::new(); - - if let Some(ar) = builder.ar(self.target) { - cfg.archiver(ar); - } - cfg.compiler(builder.cc(self.target)); - cfg.cargo_metadata(false) - .out_dir(&out_dir) - .target(&self.target.triple) - .host(&builder.config.build.triple) - .warnings(false) - .debug(false) - .opt_level(3) - .file(crtbegin_src) - .file(crtend_src); - - // Those flags are defined in src/llvm-project/compiler-rt/lib/crt/CMakeLists.txt - // Currently only consumer of those objects is musl, which use .init_array/.fini_array - // instead of .ctors/.dtors - cfg.flag("-std=c11") - .define("CRT_HAS_INITFINI_ARRAY", None) - .define("EH_USE_FRAME_REGISTRY", None); - - cfg.compile("crt"); - - t!(fs::copy(out_dir.join("crtbegin.o"), out_dir.join("crtbeginS.o"))); - t!(fs::copy(out_dir.join("crtend.o"), out_dir.join("crtendS.o"))); - out_dir - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Libunwind { - pub target: TargetSelection, -} - -impl Step for Libunwind { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/llvm-project/libunwind") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Libunwind { target: run.target }); - } - - /// Build libunwind.a - fn run(self, builder: &Builder<'_>) -> Self::Output { - builder.update_submodule(&Path::new("src/llvm-project")); - - if builder.config.dry_run() { - return PathBuf::new(); - } - - let out_dir = builder.native_dir(self.target).join("libunwind"); - let root = builder.src.join("src/llvm-project/libunwind"); - - if up_to_date(&root, &out_dir.join("libunwind.a")) { - return out_dir; - } - - let _guard = builder.msg_unstaged(Kind::Build, "libunwind.a", self.target); - t!(fs::create_dir_all(&out_dir)); - - let mut cc_cfg = cc::Build::new(); - let mut cpp_cfg = cc::Build::new(); - - cpp_cfg.cpp(true); - cpp_cfg.cpp_set_stdlib(None); - cpp_cfg.flag("-nostdinc++"); - cpp_cfg.flag("-fno-exceptions"); - cpp_cfg.flag("-fno-rtti"); - cpp_cfg.flag_if_supported("-fvisibility-global-new-delete-hidden"); - - for cfg in [&mut cc_cfg, &mut cpp_cfg].iter_mut() { - if let Some(ar) = builder.ar(self.target) { - cfg.archiver(ar); - } - cfg.target(&self.target.triple); - cfg.host(&builder.config.build.triple); - cfg.warnings(false); - cfg.debug(false); - // get_compiler() need set opt_level first. - cfg.opt_level(3); - cfg.flag("-fstrict-aliasing"); - cfg.flag("-funwind-tables"); - cfg.flag("-fvisibility=hidden"); - cfg.define("_LIBUNWIND_DISABLE_VISIBILITY_ANNOTATIONS", None); - cfg.include(root.join("include")); - cfg.cargo_metadata(false); - cfg.out_dir(&out_dir); - - if self.target.contains("x86_64-fortanix-unknown-sgx") { - cfg.static_flag(true); - cfg.flag("-fno-stack-protector"); - cfg.flag("-ffreestanding"); - cfg.flag("-fexceptions"); - - // easiest way to undefine since no API available in cc::Build to undefine - cfg.flag("-U_FORTIFY_SOURCE"); - cfg.define("_FORTIFY_SOURCE", "0"); - cfg.define("RUST_SGX", "1"); - cfg.define("__NO_STRING_INLINES", None); - cfg.define("__NO_MATH_INLINES", None); - cfg.define("_LIBUNWIND_IS_BAREMETAL", None); - cfg.define("__LIBUNWIND_IS_NATIVE_ONLY", None); - cfg.define("NDEBUG", None); - } - if self.target.contains("windows") { - cfg.define("_LIBUNWIND_HIDE_SYMBOLS", "1"); - cfg.define("_LIBUNWIND_IS_NATIVE_ONLY", "1"); - } - } - - cc_cfg.compiler(builder.cc(self.target)); - if let Ok(cxx) = builder.cxx(self.target) { - cpp_cfg.compiler(cxx); - } else { - cc_cfg.compiler(builder.cc(self.target)); - } - - // Don't set this for clang - // By default, Clang builds C code in GNU C17 mode. - // By default, Clang builds C++ code according to the C++98 standard, - // with many C++11 features accepted as extensions. - if cc_cfg.get_compiler().is_like_gnu() { - cc_cfg.flag("-std=c99"); - } - if cpp_cfg.get_compiler().is_like_gnu() { - cpp_cfg.flag("-std=c++11"); - } - - if self.target.contains("x86_64-fortanix-unknown-sgx") || self.target.contains("musl") { - // use the same GCC C compiler command to compile C++ code so we do not need to setup the - // C++ compiler env variables on the builders. - // Don't set this for clang++, as clang++ is able to compile this without libc++. - if cpp_cfg.get_compiler().is_like_gnu() { - cpp_cfg.cpp(false); - cpp_cfg.compiler(builder.cc(self.target)); - } - } - - let mut c_sources = vec![ - "Unwind-sjlj.c", - "UnwindLevel1-gcc-ext.c", - "UnwindLevel1.c", - "UnwindRegistersRestore.S", - "UnwindRegistersSave.S", - ]; - - let cpp_sources = vec!["Unwind-EHABI.cpp", "Unwind-seh.cpp", "libunwind.cpp"]; - let cpp_len = cpp_sources.len(); - - if self.target.contains("x86_64-fortanix-unknown-sgx") { - c_sources.push("UnwindRustSgx.c"); - } - - for src in c_sources { - cc_cfg.file(root.join("src").join(src).canonicalize().unwrap()); - } - - for src in &cpp_sources { - cpp_cfg.file(root.join("src").join(src).canonicalize().unwrap()); - } - - cpp_cfg.compile("unwind-cpp"); - - // FIXME: https://github.com/alexcrichton/cc-rs/issues/545#issuecomment-679242845 - let mut count = 0; - for entry in fs::read_dir(&out_dir).unwrap() { - let file = entry.unwrap().path().canonicalize().unwrap(); - if file.is_file() && file.extension() == Some(OsStr::new("o")) { - // file name starts with "Unwind-EHABI", "Unwind-seh" or "libunwind" - let file_name = file.file_name().unwrap().to_str().expect("UTF-8 file name"); - if cpp_sources.iter().any(|f| file_name.starts_with(&f[..f.len() - 4])) { - cc_cfg.object(&file); - count += 1; - } - } - } - assert_eq!(cpp_len, count, "Can't get object files from {:?}", &out_dir); - - cc_cfg.compile("unwind"); - out_dir - } -} diff --git a/src/bootstrap/metadata.rs b/src/bootstrap/metadata.rs deleted file mode 100644 index 3b20ceac8..000000000 --- a/src/bootstrap/metadata.rs +++ /dev/null @@ -1,101 +0,0 @@ -use std::path::PathBuf; -use std::process::Command; - -use serde_derive::Deserialize; - -use crate::cache::INTERNER; -use crate::util::output; -use crate::{t, Build, Crate}; - -/// For more information, see the output of -/// -#[derive(Debug, Deserialize)] -struct Output { - packages: Vec, -} - -/// For more information, see the output of -/// -#[derive(Debug, Deserialize)] -struct Package { - name: String, - source: Option, - manifest_path: String, - dependencies: Vec, - targets: Vec, -} - -/// For more information, see the output of -/// -#[derive(Debug, Deserialize)] -struct Dependency { - name: String, - source: Option, -} - -#[derive(Debug, Deserialize)] -struct Target { - kind: Vec, -} - -/// Collects and stores package metadata of each workspace members into `build`, -/// by executing `cargo metadata` commands. -pub fn build(build: &mut Build) { - for package in workspace_members(build) { - if package.source.is_none() { - let name = INTERNER.intern_string(package.name); - let mut path = PathBuf::from(package.manifest_path); - path.pop(); - let deps = package - .dependencies - .into_iter() - .filter(|dep| dep.source.is_none()) - .map(|dep| INTERNER.intern_string(dep.name)) - .collect(); - let has_lib = package.targets.iter().any(|t| t.kind.iter().any(|k| k == "lib")); - let krate = Crate { name, deps, path, has_lib }; - let relative_path = krate.local_path(build); - build.crates.insert(name, krate); - let existing_path = build.crate_paths.insert(relative_path, name); - assert!( - existing_path.is_none(), - "multiple crates with the same path: {}", - existing_path.unwrap() - ); - } - } -} - -/// Invokes `cargo metadata` to get package metadata of each workspace member. -/// -/// Note that `src/tools/cargo` is no longer a workspace member but we still -/// treat it as one here, by invoking an additional `cargo metadata` command. -fn workspace_members(build: &Build) -> impl Iterator { - let collect_metadata = |manifest_path| { - let mut cargo = Command::new(&build.initial_cargo); - cargo - // Will read the libstd Cargo.toml - // which uses the unstable `public-dependency` feature. - .env("RUSTC_BOOTSTRAP", "1") - .arg("metadata") - .arg("--format-version") - .arg("1") - .arg("--no-deps") - .arg("--manifest-path") - .arg(build.src.join(manifest_path)); - let metadata_output = output(&mut cargo); - let Output { packages, .. } = t!(serde_json::from_str(&metadata_output)); - packages - }; - - // Collects `metadata.packages` from all workspaces. - let packages = collect_metadata("Cargo.toml"); - let cargo_packages = collect_metadata("src/tools/cargo/Cargo.toml"); - let ra_packages = collect_metadata("src/tools/rust-analyzer/Cargo.toml"); - let bootstrap_packages = collect_metadata("src/bootstrap/Cargo.toml"); - - // We only care about the root package from `src/tool/cargo` workspace. - let cargo_package = cargo_packages.into_iter().find(|pkg| pkg.name == "cargo").into_iter(); - - packages.into_iter().chain(cargo_package).chain(ra_packages).chain(bootstrap_packages) -} diff --git a/src/bootstrap/metrics.rs b/src/bootstrap/metrics.rs deleted file mode 100644 index cf8d33dfc..000000000 --- a/src/bootstrap/metrics.rs +++ /dev/null @@ -1,258 +0,0 @@ -//! This module is responsible for collecting metrics profiling information for the current build -//! and dumping it to disk as JSON, to aid investigations on build and CI performance. -//! -//! As this module requires additional dependencies not present during local builds, it's cfg'd -//! away whenever the `build.metrics` config option is not set to `true`. - -use crate::builder::{Builder, Step}; -use crate::util::t; -use crate::Build; -use build_helper::metrics::{ - JsonInvocation, JsonInvocationSystemStats, JsonNode, JsonRoot, JsonStepSystemStats, Test, - TestOutcome, TestSuite, TestSuiteMetadata, -}; -use std::cell::RefCell; -use std::fs::File; -use std::io::BufWriter; -use std::time::{Duration, Instant, SystemTime}; -use sysinfo::{CpuExt, System, SystemExt}; - -// Update this number whenever a breaking change is made to the build metrics. -// -// The output format is versioned for two reasons: -// -// - The metadata is intended to be consumed by external tooling, and exposing a format version -// helps the tools determine whether they're compatible with a metrics file. -// -// - If a developer enables build metrics in their local checkout, making a breaking change to the -// metrics format would result in a hard-to-diagnose error message when an existing metrics file -// is not compatible with the new changes. With a format version number, bootstrap can discard -// incompatible metrics files instead of appending metrics to them. -// -// Version changelog: -// -// - v0: initial version -// - v1: replaced JsonNode::Test with JsonNode::TestSuite -// -const CURRENT_FORMAT_VERSION: usize = 1; - -pub(crate) struct BuildMetrics { - state: RefCell, -} - -/// NOTE: this isn't really cloning anything, but `x suggest` doesn't need metrics so this is probably ok. -impl Clone for BuildMetrics { - fn clone(&self) -> Self { - Self::init() - } -} - -impl BuildMetrics { - pub(crate) fn init() -> Self { - let state = RefCell::new(MetricsState { - finished_steps: Vec::new(), - running_steps: Vec::new(), - - system_info: System::new(), - timer_start: None, - invocation_timer_start: Instant::now(), - invocation_start: SystemTime::now(), - }); - - BuildMetrics { state } - } - - pub(crate) fn enter_step(&self, step: &S, builder: &Builder<'_>) { - // Do not record dry runs, as they'd be duplicates of the actual steps. - if builder.config.dry_run() { - return; - } - - let mut state = self.state.borrow_mut(); - - // Consider all the stats gathered so far as the parent's. - if !state.running_steps.is_empty() { - self.collect_stats(&mut *state); - } - - state.system_info.refresh_cpu(); - state.timer_start = Some(Instant::now()); - - state.running_steps.push(StepMetrics { - type_: std::any::type_name::().into(), - debug_repr: format!("{step:?}"), - - cpu_usage_time_sec: 0.0, - duration_excluding_children_sec: Duration::ZERO, - - children: Vec::new(), - test_suites: Vec::new(), - }); - } - - pub(crate) fn exit_step(&self, builder: &Builder<'_>) { - // Do not record dry runs, as they'd be duplicates of the actual steps. - if builder.config.dry_run() { - return; - } - - let mut state = self.state.borrow_mut(); - - self.collect_stats(&mut *state); - - let step = state.running_steps.pop().unwrap(); - if state.running_steps.is_empty() { - state.finished_steps.push(step); - state.timer_start = None; - } else { - state.running_steps.last_mut().unwrap().children.push(step); - - // Start collecting again for the parent step. - state.system_info.refresh_cpu(); - state.timer_start = Some(Instant::now()); - } - } - - pub(crate) fn begin_test_suite(&self, metadata: TestSuiteMetadata, builder: &Builder<'_>) { - // Do not record dry runs, as they'd be duplicates of the actual steps. - if builder.config.dry_run() { - return; - } - - let mut state = self.state.borrow_mut(); - let step = state.running_steps.last_mut().unwrap(); - step.test_suites.push(TestSuite { metadata, tests: Vec::new() }); - } - - pub(crate) fn record_test(&self, name: &str, outcome: TestOutcome, builder: &Builder<'_>) { - // Do not record dry runs, as they'd be duplicates of the actual steps. - if builder.config.dry_run() { - return; - } - - let mut state = self.state.borrow_mut(); - let step = state.running_steps.last_mut().unwrap(); - - if let Some(test_suite) = step.test_suites.last_mut() { - test_suite.tests.push(Test { name: name.to_string(), outcome }); - } else { - panic!("metrics.record_test() called without calling metrics.begin_test_suite() first"); - } - } - - fn collect_stats(&self, state: &mut MetricsState) { - let step = state.running_steps.last_mut().unwrap(); - - let elapsed = state.timer_start.unwrap().elapsed(); - step.duration_excluding_children_sec += elapsed; - - state.system_info.refresh_cpu(); - let cpu = state.system_info.cpus().iter().map(|p| p.cpu_usage()).sum::(); - step.cpu_usage_time_sec += cpu as f64 / 100.0 * elapsed.as_secs_f64(); - } - - pub(crate) fn persist(&self, build: &Build) { - let mut state = self.state.borrow_mut(); - assert!(state.running_steps.is_empty(), "steps are still executing"); - - let dest = build.out.join("metrics.json"); - - let mut system = System::new(); - system.refresh_cpu(); - system.refresh_memory(); - - let system_stats = JsonInvocationSystemStats { - cpu_threads_count: system.cpus().len(), - cpu_model: system.cpus()[0].brand().into(), - - memory_total_bytes: system.total_memory(), - }; - let steps = std::mem::take(&mut state.finished_steps); - - // Some of our CI builds consist of multiple independent CI invocations. Ensure all the - // previous invocations are still present in the resulting file. - let mut invocations = match std::fs::read(&dest) { - Ok(contents) => { - // We first parse just the format_version field to have the check succeed even if - // the rest of the contents are not valid anymore. - let version: OnlyFormatVersion = t!(serde_json::from_slice(&contents)); - if version.format_version == CURRENT_FORMAT_VERSION { - t!(serde_json::from_slice::(&contents)).invocations - } else { - println!( - "warning: overriding existing build/metrics.json, as it's not \ - compatible with build metrics format version {CURRENT_FORMAT_VERSION}." - ); - Vec::new() - } - } - Err(err) => { - if err.kind() != std::io::ErrorKind::NotFound { - panic!("failed to open existing metrics file at {}: {err}", dest.display()); - } - Vec::new() - } - }; - invocations.push(JsonInvocation { - start_time: state - .invocation_start - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap() - .as_secs(), - duration_including_children_sec: state.invocation_timer_start.elapsed().as_secs_f64(), - children: steps.into_iter().map(|step| self.prepare_json_step(step)).collect(), - }); - - let json = JsonRoot { format_version: CURRENT_FORMAT_VERSION, system_stats, invocations }; - - t!(std::fs::create_dir_all(dest.parent().unwrap())); - let mut file = BufWriter::new(t!(File::create(&dest))); - t!(serde_json::to_writer(&mut file, &json)); - } - - fn prepare_json_step(&self, step: StepMetrics) -> JsonNode { - let mut children = Vec::new(); - children.extend(step.children.into_iter().map(|child| self.prepare_json_step(child))); - children.extend(step.test_suites.into_iter().map(JsonNode::TestSuite)); - - JsonNode::RustbuildStep { - type_: step.type_, - debug_repr: step.debug_repr, - - duration_excluding_children_sec: step.duration_excluding_children_sec.as_secs_f64(), - system_stats: JsonStepSystemStats { - cpu_utilization_percent: step.cpu_usage_time_sec * 100.0 - / step.duration_excluding_children_sec.as_secs_f64(), - }, - - children, - } - } -} - -struct MetricsState { - finished_steps: Vec, - running_steps: Vec, - - system_info: System, - timer_start: Option, - invocation_timer_start: Instant, - invocation_start: SystemTime, -} - -struct StepMetrics { - type_: String, - debug_repr: String, - - cpu_usage_time_sec: f64, - duration_excluding_children_sec: Duration, - - children: Vec, - test_suites: Vec, -} - -#[derive(serde_derive::Deserialize)] -struct OnlyFormatVersion { - #[serde(default)] // For version 0 the field was not present. - format_version: usize, -} diff --git a/src/bootstrap/render_tests.rs b/src/bootstrap/render_tests.rs deleted file mode 100644 index 6802bf451..000000000 --- a/src/bootstrap/render_tests.rs +++ /dev/null @@ -1,400 +0,0 @@ -//! This module renders the JSON output of libtest into a human-readable form, trying to be as -//! similar to libtest's native output as possible. -//! -//! This is needed because we need to use libtest in JSON mode to extract granular information -//! about the executed tests. Doing so suppresses the human-readable output, and (compared to Cargo -//! and rustc) libtest doesn't include the rendered human-readable output as a JSON field. We had -//! to reimplement all the rendering logic in this module because of that. - -use crate::builder::Builder; -use std::io::{BufRead, BufReader, Read, Write}; -use std::process::{ChildStdout, Command, Stdio}; -use std::time::Duration; -use termcolor::{Color, ColorSpec, WriteColor}; - -const TERSE_TESTS_PER_LINE: usize = 88; - -pub(crate) fn add_flags_and_try_run_tests(builder: &Builder<'_>, cmd: &mut Command) -> bool { - if cmd.get_args().position(|arg| arg == "--").is_none() { - cmd.arg("--"); - } - cmd.args(&["-Z", "unstable-options", "--format", "json"]); - - try_run_tests(builder, cmd, false) -} - -pub(crate) fn try_run_tests(builder: &Builder<'_>, cmd: &mut Command, stream: bool) -> bool { - if builder.config.dry_run() { - return true; - } - - if !run_tests(builder, cmd, stream) { - if builder.fail_fast { - crate::exit!(1); - } else { - let mut failures = builder.delayed_failures.borrow_mut(); - failures.push(format!("{cmd:?}")); - false - } - } else { - true - } -} - -fn run_tests(builder: &Builder<'_>, cmd: &mut Command, stream: bool) -> bool { - cmd.stdout(Stdio::piped()); - - builder.verbose(&format!("running: {cmd:?}")); - - let mut process = cmd.spawn().unwrap(); - - // This runs until the stdout of the child is closed, which means the child exited. We don't - // run this on another thread since the builder is not Sync. - let renderer = Renderer::new(process.stdout.take().unwrap(), builder); - if stream { - renderer.stream_all(); - } else { - renderer.render_all(); - } - - let result = process.wait_with_output().unwrap(); - if !result.status.success() && builder.is_verbose() { - println!( - "\n\ncommand did not execute successfully: {cmd:?}\n\ - expected success, got: {}", - result.status - ); - } - - result.status.success() -} - -struct Renderer<'a> { - stdout: BufReader, - failures: Vec, - benches: Vec, - builder: &'a Builder<'a>, - tests_count: Option, - executed_tests: usize, - terse_tests_in_line: usize, -} - -impl<'a> Renderer<'a> { - fn new(stdout: ChildStdout, builder: &'a Builder<'a>) -> Self { - Self { - stdout: BufReader::new(stdout), - benches: Vec::new(), - failures: Vec::new(), - builder, - tests_count: None, - executed_tests: 0, - terse_tests_in_line: 0, - } - } - - fn render_all(mut self) { - let mut line = Vec::new(); - loop { - line.clear(); - match self.stdout.read_until(b'\n', &mut line) { - Ok(_) => {} - Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => break, - Err(err) => panic!("failed to read output of test runner: {err}"), - } - if line.is_empty() { - break; - } - - match serde_json::from_slice(&line) { - Ok(parsed) => self.render_message(parsed), - Err(_err) => { - // Handle non-JSON output, for example when --nocapture is passed. - let mut stdout = std::io::stdout(); - stdout.write_all(&line).unwrap(); - let _ = stdout.flush(); - } - } - } - } - - /// Renders the stdout characters one by one - fn stream_all(mut self) { - let mut buffer = [0; 1]; - loop { - match self.stdout.read(&mut buffer) { - Ok(0) => break, - Ok(_) => { - let mut stdout = std::io::stdout(); - stdout.write_all(&buffer).unwrap(); - let _ = stdout.flush(); - } - Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => break, - Err(err) => panic!("failed to read output of test runner: {err}"), - } - } - } - - fn render_test_outcome(&mut self, outcome: Outcome<'_>, test: &TestOutcome) { - self.executed_tests += 1; - - #[cfg(feature = "build-metrics")] - self.builder.metrics.record_test( - &test.name, - match outcome { - Outcome::Ok | Outcome::BenchOk => build_helper::metrics::TestOutcome::Passed, - Outcome::Failed => build_helper::metrics::TestOutcome::Failed, - Outcome::Ignored { reason } => build_helper::metrics::TestOutcome::Ignored { - ignore_reason: reason.map(|s| s.to_string()), - }, - }, - self.builder, - ); - - if self.builder.config.verbose_tests { - self.render_test_outcome_verbose(outcome, test); - } else { - self.render_test_outcome_terse(outcome, test); - } - } - - fn render_test_outcome_verbose(&self, outcome: Outcome<'_>, test: &TestOutcome) { - print!("test {} ... ", test.name); - self.builder.colored_stdout(|stdout| outcome.write_long(stdout)).unwrap(); - if let Some(exec_time) = test.exec_time { - print!(" ({exec_time:.2?})"); - } - println!(); - } - - fn render_test_outcome_terse(&mut self, outcome: Outcome<'_>, _: &TestOutcome) { - if self.terse_tests_in_line != 0 && self.terse_tests_in_line % TERSE_TESTS_PER_LINE == 0 { - if let Some(total) = self.tests_count { - let total = total.to_string(); - let executed = format!("{:>width$}", self.executed_tests - 1, width = total.len()); - print!(" {executed}/{total}"); - } - println!(); - self.terse_tests_in_line = 0; - } - - self.terse_tests_in_line += 1; - self.builder.colored_stdout(|stdout| outcome.write_short(stdout)).unwrap(); - let _ = std::io::stdout().flush(); - } - - fn render_suite_outcome(&self, outcome: Outcome<'_>, suite: &SuiteOutcome) { - // The terse output doesn't end with a newline, so we need to add it ourselves. - if !self.builder.config.verbose_tests { - println!(); - } - - if !self.failures.is_empty() { - println!("\nfailures:\n"); - for failure in &self.failures { - if failure.stdout.is_some() || failure.message.is_some() { - println!("---- {} stdout ----", failure.name); - if let Some(stdout) = &failure.stdout { - println!("{stdout}"); - } - if let Some(message) = &failure.message { - println!("note: {message}"); - } - } - } - - println!("\nfailures:"); - for failure in &self.failures { - println!(" {}", failure.name); - } - } - - if !self.benches.is_empty() { - println!("\nbenchmarks:"); - - let mut rows = Vec::new(); - for bench in &self.benches { - rows.push(( - &bench.name, - format!("{:.2?}/iter", Duration::from_nanos(bench.median)), - format!("+/- {:.2?}", Duration::from_nanos(bench.deviation)), - )); - } - - let max_0 = rows.iter().map(|r| r.0.len()).max().unwrap_or(0); - let max_1 = rows.iter().map(|r| r.1.len()).max().unwrap_or(0); - let max_2 = rows.iter().map(|r| r.2.len()).max().unwrap_or(0); - for row in &rows { - println!(" {:max_1$} {:>max_2$}", row.0, row.1, row.2); - } - } - - print!("\ntest result: "); - self.builder.colored_stdout(|stdout| outcome.write_long(stdout)).unwrap(); - println!( - ". {} passed; {} failed; {} ignored; {} measured; {} filtered out; \ - finished in {:.2?}\n", - suite.passed, - suite.failed, - suite.ignored, - suite.measured, - suite.filtered_out, - Duration::from_secs_f64(suite.exec_time) - ); - } - - fn render_message(&mut self, message: Message) { - match message { - Message::Suite(SuiteMessage::Started { test_count }) => { - println!("\nrunning {test_count} tests"); - self.executed_tests = 0; - self.terse_tests_in_line = 0; - self.tests_count = Some(test_count); - } - Message::Suite(SuiteMessage::Ok(outcome)) => { - self.render_suite_outcome(Outcome::Ok, &outcome); - } - Message::Suite(SuiteMessage::Failed(outcome)) => { - self.render_suite_outcome(Outcome::Failed, &outcome); - } - Message::Bench(outcome) => { - // The formatting for benchmarks doesn't replicate 1:1 the formatting libtest - // outputs, mostly because libtest's formatting is broken in terse mode, which is - // the default used by our monorepo. We use a different formatting instead: - // successful benchmarks are just showed as "benchmarked"/"b", and the details are - // outputted at the bottom like failures. - let fake_test_outcome = TestOutcome { - name: outcome.name.clone(), - exec_time: None, - stdout: None, - message: None, - }; - self.render_test_outcome(Outcome::BenchOk, &fake_test_outcome); - self.benches.push(outcome); - } - Message::Test(TestMessage::Ok(outcome)) => { - self.render_test_outcome(Outcome::Ok, &outcome); - } - Message::Test(TestMessage::Ignored(outcome)) => { - self.render_test_outcome( - Outcome::Ignored { reason: outcome.message.as_deref() }, - &outcome, - ); - } - Message::Test(TestMessage::Failed(outcome)) => { - self.render_test_outcome(Outcome::Failed, &outcome); - self.failures.push(outcome); - } - Message::Test(TestMessage::Timeout { name }) => { - println!("test {name} has been running for a long time"); - } - Message::Test(TestMessage::Started) => {} // Not useful - } - } -} - -enum Outcome<'a> { - Ok, - BenchOk, - Failed, - Ignored { reason: Option<&'a str> }, -} - -impl Outcome<'_> { - fn write_short(&self, writer: &mut dyn WriteColor) -> Result<(), std::io::Error> { - match self { - Outcome::Ok => { - writer.set_color(&ColorSpec::new().set_fg(Some(Color::Green)))?; - write!(writer, ".")?; - } - Outcome::BenchOk => { - writer.set_color(&ColorSpec::new().set_fg(Some(Color::Cyan)))?; - write!(writer, "b")?; - } - Outcome::Failed => { - writer.set_color(&ColorSpec::new().set_fg(Some(Color::Red)))?; - write!(writer, "F")?; - } - Outcome::Ignored { .. } => { - writer.set_color(&ColorSpec::new().set_fg(Some(Color::Yellow)))?; - write!(writer, "i")?; - } - } - writer.reset() - } - - fn write_long(&self, writer: &mut dyn WriteColor) -> Result<(), std::io::Error> { - match self { - Outcome::Ok => { - writer.set_color(&ColorSpec::new().set_fg(Some(Color::Green)))?; - write!(writer, "ok")?; - } - Outcome::BenchOk => { - writer.set_color(&ColorSpec::new().set_fg(Some(Color::Cyan)))?; - write!(writer, "benchmarked")?; - } - Outcome::Failed => { - writer.set_color(&ColorSpec::new().set_fg(Some(Color::Red)))?; - write!(writer, "FAILED")?; - } - Outcome::Ignored { reason } => { - writer.set_color(&ColorSpec::new().set_fg(Some(Color::Yellow)))?; - write!(writer, "ignored")?; - if let Some(reason) = reason { - write!(writer, ", {reason}")?; - } - } - } - writer.reset() - } -} - -#[derive(serde_derive::Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -enum Message { - Suite(SuiteMessage), - Test(TestMessage), - Bench(BenchOutcome), -} - -#[derive(serde_derive::Deserialize)] -#[serde(tag = "event", rename_all = "snake_case")] -enum SuiteMessage { - Ok(SuiteOutcome), - Failed(SuiteOutcome), - Started { test_count: usize }, -} - -#[derive(serde_derive::Deserialize)] -struct SuiteOutcome { - passed: usize, - failed: usize, - ignored: usize, - measured: usize, - filtered_out: usize, - exec_time: f64, -} - -#[derive(serde_derive::Deserialize)] -#[serde(tag = "event", rename_all = "snake_case")] -enum TestMessage { - Ok(TestOutcome), - Failed(TestOutcome), - Ignored(TestOutcome), - Timeout { name: String }, - Started, -} - -#[derive(serde_derive::Deserialize)] -struct BenchOutcome { - name: String, - median: u64, - deviation: u64, -} - -#[derive(serde_derive::Deserialize)] -struct TestOutcome { - name: String, - exec_time: Option, - stdout: Option, - message: Option, -} diff --git a/src/bootstrap/run.rs b/src/bootstrap/run.rs deleted file mode 100644 index 4082f5bb9..000000000 --- a/src/bootstrap/run.rs +++ /dev/null @@ -1,297 +0,0 @@ -use std::path::PathBuf; -use std::process::Command; - -use clap_complete::shells; - -use crate::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::config::TargetSelection; -use crate::dist::distdir; -use crate::flags::get_completion; -use crate::test; -use crate::tool::{self, SourceType, Tool}; -use crate::util::output; -use crate::Mode; - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct ExpandYamlAnchors; - -impl Step for ExpandYamlAnchors { - type Output = (); - - /// Runs the `expand-yaml_anchors` tool. - /// - /// This tool in `src/tools` reads the CI configuration files written in YAML and expands the - /// anchors in them, since GitHub Actions doesn't support them. - fn run(self, builder: &Builder<'_>) { - builder.info("Expanding YAML anchors in the GitHub Actions configuration"); - builder.run_delaying_failure( - &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("generate").arg(&builder.src), - ); - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/expand-yaml-anchors") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(ExpandYamlAnchors); - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct BuildManifest; - -impl Step for BuildManifest { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/build-manifest") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(BuildManifest); - } - - fn run(self, builder: &Builder<'_>) { - // This gets called by `promote-release` - // (https://github.com/rust-lang/promote-release). - let mut cmd = builder.tool_cmd(Tool::BuildManifest); - let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| { - panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n") - }); - let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| { - panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n") - }); - - let today = output(Command::new("date").arg("+%Y-%m-%d")); - - cmd.arg(sign); - cmd.arg(distdir(builder)); - cmd.arg(today.trim()); - cmd.arg(addr); - cmd.arg(&builder.config.channel); - - builder.create_dir(&distdir(builder)); - builder.run(&mut cmd); - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct BumpStage0; - -impl Step for BumpStage0 { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/bump-stage0") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(BumpStage0); - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - let mut cmd = builder.tool_cmd(Tool::BumpStage0); - cmd.args(builder.config.args()); - builder.run(&mut cmd); - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct ReplaceVersionPlaceholder; - -impl Step for ReplaceVersionPlaceholder { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/replace-version-placeholder") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(ReplaceVersionPlaceholder); - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - let mut cmd = builder.tool_cmd(Tool::ReplaceVersionPlaceholder); - cmd.arg(&builder.src); - builder.run(&mut cmd); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Miri { - stage: u32, - host: TargetSelection, - target: TargetSelection, -} - -impl Step for Miri { - type Output = (); - const ONLY_HOSTS: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/miri") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Miri { - stage: run.builder.top_stage, - host: run.build_triple(), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let host = self.host; - let target = self.target; - let compiler = builder.compiler(stage, host); - - let miri = builder - .ensure(tool::Miri { compiler, target: self.host, extra_features: Vec::new() }) - .expect("in-tree tool"); - let miri_sysroot = test::Miri::build_miri_sysroot(builder, compiler, &miri, target); - - // # Run miri. - // Running it via `cargo run` as that figures out the right dylib path. - // add_rustc_lib_path does not add the path that contains librustc_driver-<...>.so. - let mut miri = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - host, - "run", - "src/tools/miri", - SourceType::InTree, - &[], - ); - miri.add_rustc_lib_path(builder, compiler); - // Forward arguments. - miri.arg("--").arg("--target").arg(target.rustc_target_arg()); - miri.args(builder.config.args()); - - // miri tests need to know about the stage sysroot - miri.env("MIRI_SYSROOT", &miri_sysroot); - - let mut miri = Command::from(miri); - builder.run(&mut miri); - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct CollectLicenseMetadata; - -impl Step for CollectLicenseMetadata { - type Output = PathBuf; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/collect-license-metadata") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(CollectLicenseMetadata); - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - let Some(reuse) = &builder.config.reuse else { - panic!("REUSE is required to collect the license metadata"); - }; - - // Temporary location, it will be moved to src/etc once it's accurate. - let dest = builder.out.join("license-metadata.json"); - - let mut cmd = builder.tool_cmd(Tool::CollectLicenseMetadata); - cmd.env("REUSE_EXE", reuse); - cmd.env("DEST", &dest); - builder.run(&mut cmd); - - dest - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct GenerateCopyright; - -impl Step for GenerateCopyright { - type Output = PathBuf; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/generate-copyright") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(GenerateCopyright); - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - let license_metadata = builder.ensure(CollectLicenseMetadata); - - // Temporary location, it will be moved to the proper one once it's accurate. - let dest = builder.out.join("COPYRIGHT.md"); - - let mut cmd = builder.tool_cmd(Tool::GenerateCopyright); - cmd.env("LICENSE_METADATA", &license_metadata); - cmd.env("DEST", &dest); - builder.run(&mut cmd); - - dest - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct GenerateWindowsSys; - -impl Step for GenerateWindowsSys { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/generate-windows-sys") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(GenerateWindowsSys); - } - - fn run(self, builder: &Builder<'_>) { - let mut cmd = builder.tool_cmd(Tool::GenerateWindowsSys); - cmd.arg(&builder.src); - builder.run(&mut cmd); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct GenerateCompletions; - -impl Step for GenerateCompletions { - type Output = (); - - /// Uses `clap_complete` to generate shell completions. - fn run(self, builder: &Builder<'_>) { - // FIXME(clubby789): enable zsh when clap#4898 is fixed - let [bash, fish, powershell] = ["x.py.sh", "x.py.fish", "x.py.ps1"] - .map(|filename| builder.src.join("src/etc/completions").join(filename)); - if let Some(comp) = get_completion(shells::Bash, &bash) { - std::fs::write(&bash, comp).expect("writing bash completion"); - } - if let Some(comp) = get_completion(shells::Fish, &fish) { - std::fs::write(&fish, comp).expect("writing fish completion"); - } - if let Some(comp) = get_completion(shells::PowerShell, &powershell) { - std::fs::write(&powershell, comp).expect("writing powershell completion"); - } - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("generate-completions") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(GenerateCompletions); - } -} diff --git a/src/bootstrap/sanity.rs b/src/bootstrap/sanity.rs deleted file mode 100644 index 0febdf250..000000000 --- a/src/bootstrap/sanity.rs +++ /dev/null @@ -1,267 +0,0 @@ -//! Sanity checking performed by rustbuild before actually executing anything. -//! -//! This module contains the implementation of ensuring that the build -//! environment looks reasonable before progressing. This will verify that -//! various programs like git and python exist, along with ensuring that all C -//! compilers for cross-compiling are found. -//! -//! In theory if we get past this phase it's a bug if a build fails, but in -//! practice that's likely not true! - -use std::collections::HashMap; -use std::env; -use std::ffi::{OsStr, OsString}; -use std::fs; -use std::path::PathBuf; -use std::process::Command; - -use crate::cache::INTERNER; -use crate::config::Target; -use crate::util::output; -use crate::Build; - -pub struct Finder { - cache: HashMap>, - path: OsString, -} - -impl Finder { - pub fn new() -> Self { - Self { cache: HashMap::new(), path: env::var_os("PATH").unwrap_or_default() } - } - - pub fn maybe_have>(&mut self, cmd: S) -> Option { - let cmd: OsString = cmd.into(); - let path = &self.path; - self.cache - .entry(cmd.clone()) - .or_insert_with(|| { - for path in env::split_paths(path) { - let target = path.join(&cmd); - let mut cmd_exe = cmd.clone(); - cmd_exe.push(".exe"); - - if target.is_file() // some/path/git - || path.join(&cmd_exe).exists() // some/path/git.exe - || target.join(&cmd_exe).exists() - // some/path/git/git.exe - { - return Some(target); - } - } - None - }) - .clone() - } - - pub fn must_have>(&mut self, cmd: S) -> PathBuf { - self.maybe_have(&cmd).unwrap_or_else(|| { - panic!("\n\ncouldn't find required command: {:?}\n\n", cmd.as_ref()); - }) - } -} - -pub fn check(build: &mut Build) { - let skip_target_sanity = - env::var_os("BOOTSTRAP_SKIP_TARGET_SANITY").is_some_and(|s| s == "1" || s == "true"); - - let path = env::var_os("PATH").unwrap_or_default(); - // On Windows, quotes are invalid characters for filename paths, and if - // one is present as part of the PATH then that can lead to the system - // being unable to identify the files properly. See - // https://github.com/rust-lang/rust/issues/34959 for more details. - if cfg!(windows) && path.to_string_lossy().contains('\"') { - panic!("PATH contains invalid character '\"'"); - } - - let mut cmd_finder = Finder::new(); - // If we've got a git directory we're gonna need git to update - // submodules and learn about various other aspects. - if build.rust_info().is_managed_git_subrepository() { - cmd_finder.must_have("git"); - } - - // We need cmake, but only if we're actually building LLVM or sanitizers. - let building_llvm = build.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) - && build - .hosts - .iter() - .map(|host| { - build - .config - .target_config - .get(host) - .map(|config| config.llvm_config.is_none()) - .unwrap_or(true) - }) - .any(|build_llvm_ourselves| build_llvm_ourselves); - - let need_cmake = building_llvm || build.config.any_sanitizers_enabled(); - if need_cmake && cmd_finder.maybe_have("cmake").is_none() { - eprintln!( - " -Couldn't find required command: cmake - -You should install cmake, or set `download-ci-llvm = true` in the -`[llvm]` section of `config.toml` to download LLVM rather -than building it. -" - ); - crate::exit!(1); - } - - build.config.python = build - .config - .python - .take() - .map(|p| cmd_finder.must_have(p)) - .or_else(|| env::var_os("BOOTSTRAP_PYTHON").map(PathBuf::from)) // set by bootstrap.py - .or_else(|| cmd_finder.maybe_have("python")) - .or_else(|| cmd_finder.maybe_have("python3")) - .or_else(|| cmd_finder.maybe_have("python2")); - - build.config.nodejs = build - .config - .nodejs - .take() - .map(|p| cmd_finder.must_have(p)) - .or_else(|| cmd_finder.maybe_have("node")) - .or_else(|| cmd_finder.maybe_have("nodejs")); - - build.config.npm = build - .config - .npm - .take() - .map(|p| cmd_finder.must_have(p)) - .or_else(|| cmd_finder.maybe_have("npm")); - - build.config.gdb = build - .config - .gdb - .take() - .map(|p| cmd_finder.must_have(p)) - .or_else(|| cmd_finder.maybe_have("gdb")); - - build.config.reuse = build - .config - .reuse - .take() - .map(|p| cmd_finder.must_have(p)) - .or_else(|| cmd_finder.maybe_have("reuse")); - - // We're gonna build some custom C code here and there, host triples - // also build some C++ shims for LLVM so we need a C++ compiler. - for target in &build.targets { - // On emscripten we don't actually need the C compiler to just - // build the target artifacts, only for testing. For the sake - // of easier bot configuration, just skip detection. - if target.contains("emscripten") { - continue; - } - - // We don't use a C compiler on wasm32 - if target.contains("wasm32") { - continue; - } - - // Some environments don't want or need these tools, such as when testing Miri. - // FIXME: it would be better to refactor this code to split necessary setup from pure sanity - // checks, and have a regular flag for skipping the latter. Also see - // . - if skip_target_sanity { - continue; - } - - if !build.config.dry_run() { - cmd_finder.must_have(build.cc(*target)); - if let Some(ar) = build.ar(*target) { - cmd_finder.must_have(ar); - } - } - } - - for host in &build.hosts { - if !build.config.dry_run() { - cmd_finder.must_have(build.cxx(*host).unwrap()); - } - } - - if build.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) { - // Externally configured LLVM requires FileCheck to exist - let filecheck = build.llvm_filecheck(build.build); - if !filecheck.starts_with(&build.out) && !filecheck.exists() && build.config.codegen_tests { - panic!("FileCheck executable {filecheck:?} does not exist"); - } - } - - for target in &build.targets { - build - .config - .target_config - .entry(*target) - .or_insert_with(|| Target::from_triple(&target.triple)); - - if (target.contains("-none-") || target.contains("nvptx")) - && build.no_std(*target) == Some(false) - { - panic!("All the *-none-* and nvptx* targets are no-std targets") - } - - // Some environments don't want or need these tools, such as when testing Miri. - // FIXME: it would be better to refactor this code to split necessary setup from pure sanity - // checks, and have a regular flag for skipping the latter. Also see - // . - if skip_target_sanity { - continue; - } - - // Make sure musl-root is valid. - if target.contains("musl") && !target.contains("unikraft") { - // If this is a native target (host is also musl) and no musl-root is given, - // fall back to the system toolchain in /usr before giving up - if build.musl_root(*target).is_none() && build.config.build == *target { - let target = build.config.target_config.entry(*target).or_default(); - target.musl_root = Some("/usr".into()); - } - match build.musl_libdir(*target) { - Some(libdir) => { - if fs::metadata(libdir.join("libc.a")).is_err() { - panic!("couldn't find libc.a in musl libdir: {}", libdir.display()); - } - } - None => panic!( - "when targeting MUSL either the rust.musl-root \ - option or the target.$TARGET.musl-root option must \ - be specified in config.toml" - ), - } - } - - if need_cmake && target.contains("msvc") { - // There are three builds of cmake on windows: MSVC, MinGW, and - // Cygwin. The Cygwin build does not have generators for Visual - // Studio, so detect that here and error. - let out = output(Command::new("cmake").arg("--help")); - if !out.contains("Visual Studio") { - panic!( - " -cmake does not support Visual Studio generators. - -This is likely due to it being an msys/cygwin build of cmake, -rather than the required windows version, built using MinGW -or Visual Studio. - -If you are building under msys2 try installing the mingw-w64-x86_64-cmake -package instead of cmake: - -$ pacman -R cmake && pacman -S mingw-w64-x86_64-cmake -" - ); - } - } - } - - if let Some(ref s) = build.config.ccache { - cmd_finder.must_have(s); - } -} diff --git a/src/bootstrap/setup.rs b/src/bootstrap/setup.rs deleted file mode 100644 index ef0234957..000000000 --- a/src/bootstrap/setup.rs +++ /dev/null @@ -1,599 +0,0 @@ -use crate::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::Config; -use crate::{t, VERSION}; -use sha2::Digest; -use std::env::consts::EXE_SUFFIX; -use std::fmt::Write as _; -use std::fs::File; -use std::io::Write; -use std::path::{Path, PathBuf, MAIN_SEPARATOR}; -use std::process::Command; -use std::str::FromStr; -use std::{fmt, fs, io}; - -#[cfg(test)] -mod tests; - -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] -pub enum Profile { - Compiler, - Codegen, - Library, - Tools, - Dist, - None, -} - -/// A list of historical hashes of `src/etc/rust_analyzer_settings.json`. -/// New entries should be appended whenever this is updated so we can detect -/// outdated vs. user-modified settings files. -static SETTINGS_HASHES: &[&str] = &[ - "ea67e259dedf60d4429b6c349a564ffcd1563cf41c920a856d1f5b16b4701ac8", - "56e7bf011c71c5d81e0bf42e84938111847a810eee69d906bba494ea90b51922", - "af1b5efe196aed007577899db9dae15d6dbc923d6fa42fa0934e68617ba9bbe0", - "3468fea433c25fff60be6b71e8a215a732a7b1268b6a83bf10d024344e140541", - "47d227f424bf889b0d899b9cc992d5695e1b78c406e183cd78eafefbe5488923", - "b526bd58d0262dd4dda2bff5bc5515b705fb668a46235ace3e057f807963a11a", -]; -static RUST_ANALYZER_SETTINGS: &str = include_str!("../etc/rust_analyzer_settings.json"); - -impl Profile { - fn include_path(&self, src_path: &Path) -> PathBuf { - PathBuf::from(format!("{}/src/bootstrap/defaults/config.{}.toml", src_path.display(), self)) - } - - pub fn all() -> impl Iterator { - use Profile::*; - // N.B. these are ordered by how they are displayed, not alphabetically - [Library, Compiler, Codegen, Tools, Dist, None].iter().copied() - } - - pub fn purpose(&self) -> String { - use Profile::*; - match self { - Library => "Contribute to the standard library", - Compiler => "Contribute to the compiler itself", - Codegen => "Contribute to the compiler, and also modify LLVM or codegen", - Tools => "Contribute to tools which depend on the compiler, but do not modify it directly (e.g. rustdoc, clippy, miri)", - Dist => "Install Rust from source", - None => "Do not modify `config.toml`" - } - .to_string() - } - - pub fn all_for_help(indent: &str) -> String { - let mut out = String::new(); - for choice in Profile::all() { - writeln!(&mut out, "{}{}: {}", indent, choice, choice.purpose()).unwrap(); - } - out - } - - pub fn as_str(&self) -> &'static str { - match self { - Profile::Compiler => "compiler", - Profile::Codegen => "codegen", - Profile::Library => "library", - Profile::Tools => "tools", - Profile::Dist => "dist", - Profile::None => "none", - } - } -} - -impl FromStr for Profile { - type Err = String; - - fn from_str(s: &str) -> Result { - match s { - "lib" | "library" => Ok(Profile::Library), - "compiler" => Ok(Profile::Compiler), - "llvm" | "codegen" => Ok(Profile::Codegen), - "maintainer" | "dist" | "user" => Ok(Profile::Dist), - "tools" | "tool" | "rustdoc" | "clippy" | "miri" | "rustfmt" | "rls" => { - Ok(Profile::Tools) - } - "none" => Ok(Profile::None), - _ => Err(format!("unknown profile: '{s}'")), - } - } -} - -impl fmt::Display for Profile { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.as_str()) - } -} - -impl Step for Profile { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(mut run: ShouldRun<'_>) -> ShouldRun<'_> { - for choice in Profile::all() { - run = run.alias(choice.as_str()); - } - run - } - - fn make_run(run: RunConfig<'_>) { - if run.builder.config.dry_run() { - return; - } - - // for Profile, `run.paths` will have 1 and only 1 element - // this is because we only accept at most 1 path from user input. - // If user calls `x.py setup` without arguments, the interactive TUI - // will guide user to provide one. - let profile = if run.paths.len() > 1 { - // HACK: `builder` runs this step with all paths if no path was passed. - t!(interactive_path()) - } else { - run.paths - .first() - .unwrap() - .assert_single_path() - .path - .as_path() - .as_os_str() - .to_str() - .unwrap() - .parse() - .unwrap() - }; - - run.builder.ensure(profile); - } - - fn run(self, builder: &Builder<'_>) { - setup(&builder.build.config, self) - } -} - -pub fn setup(config: &Config, profile: Profile) { - let suggestions: &[&str] = match profile { - Profile::Codegen | Profile::Compiler | Profile::None => &["check", "build", "test"], - Profile::Tools => &[ - "check", - "build", - "test tests/rustdoc*", - "test src/tools/clippy", - "test src/tools/miri", - "test src/tools/rustfmt", - ], - Profile::Library => &["check", "build", "test library/std", "doc"], - Profile::Dist => &["dist", "build"], - }; - - println!(); - - println!("To get started, try one of the following commands:"); - for cmd in suggestions { - println!("- `x.py {cmd}`"); - } - - if profile != Profile::Dist { - println!( - "For more suggestions, see https://rustc-dev-guide.rust-lang.org/building/suggested.html" - ); - } - - if profile == Profile::Tools { - eprintln!(); - eprintln!( - "note: the `tools` profile sets up the `stage2` toolchain (use \ - `rustup toolchain link 'name' host/build/stage2` to use rustc)" - ) - } - - let path = &config.config.clone().unwrap_or(PathBuf::from("config.toml")); - setup_config_toml(path, profile, config); -} - -fn setup_config_toml(path: &PathBuf, profile: Profile, config: &Config) { - if profile == Profile::None { - return; - } - if path.exists() { - eprintln!(); - eprintln!( - "error: you asked `x.py` to setup a new config file, but one already exists at `{}`", - path.display() - ); - eprintln!("help: try adding `profile = \"{}\"` at the top of {}", profile, path.display()); - eprintln!( - "note: this will use the configuration in {}", - profile.include_path(&config.src).display() - ); - crate::exit!(1); - } - - let settings = format!( - "# Includes one of the default files in src/bootstrap/defaults\n\ - profile = \"{profile}\"\n\ - changelog-seen = {VERSION}\n" - ); - - t!(fs::write(path, settings)); - - let include_path = profile.include_path(&config.src); - println!("`x.py` will now use the configuration at {}", include_path.display()); -} - -/// Creates a toolchain link for stage1 using `rustup` -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] -pub struct Link; -impl Step for Link { - type Output = (); - const DEFAULT: bool = true; - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("link") - } - fn make_run(run: RunConfig<'_>) { - if run.builder.config.dry_run() { - return; - } - if let [cmd] = &run.paths[..] { - if cmd.assert_single_path().path.as_path().as_os_str() == "link" { - run.builder.ensure(Link); - } - } - } - fn run(self, builder: &Builder<'_>) -> Self::Output { - let config = &builder.config; - if config.dry_run() { - return; - } - let stage_path = - ["build", config.build.rustc_target_arg(), "stage1"].join(&MAIN_SEPARATOR.to_string()); - - if !rustup_installed() { - eprintln!("`rustup` is not installed; cannot link `stage1` toolchain"); - } else if stage_dir_exists(&stage_path[..]) && !config.dry_run() { - attempt_toolchain_link(&stage_path[..]); - } - } -} - -fn rustup_installed() -> bool { - Command::new("rustup") - .arg("--version") - .stdout(std::process::Stdio::null()) - .output() - .map_or(false, |output| output.status.success()) -} - -fn stage_dir_exists(stage_path: &str) -> bool { - match fs::create_dir(&stage_path) { - Ok(_) => true, - Err(_) => Path::new(&stage_path).exists(), - } -} - -fn attempt_toolchain_link(stage_path: &str) { - if toolchain_is_linked() { - return; - } - - if !ensure_stage1_toolchain_placeholder_exists(stage_path) { - eprintln!( - "Failed to create a template for stage 1 toolchain or confirm that it already exists" - ); - return; - } - - if try_link_toolchain(&stage_path) { - println!( - "Added `stage1` rustup toolchain; try `cargo +stage1 build` on a separate rust project to run a newly-built toolchain" - ); - } else { - eprintln!("`rustup` failed to link stage 1 build to `stage1` toolchain"); - eprintln!( - "To manually link stage 1 build to `stage1` toolchain, run:\n - `rustup toolchain link stage1 {}`", - &stage_path - ); - } -} - -fn toolchain_is_linked() -> bool { - match Command::new("rustup") - .args(&["toolchain", "list"]) - .stdout(std::process::Stdio::piped()) - .output() - { - Ok(toolchain_list) => { - if !String::from_utf8_lossy(&toolchain_list.stdout).contains("stage1") { - return false; - } - // The toolchain has already been linked. - println!( - "`stage1` toolchain already linked; not attempting to link `stage1` toolchain" - ); - } - Err(_) => { - // In this case, we don't know if the `stage1` toolchain has been linked; - // but `rustup` failed, so let's not go any further. - println!( - "`rustup` failed to list current toolchains; not attempting to link `stage1` toolchain" - ); - } - } - true -} - -fn try_link_toolchain(stage_path: &str) -> bool { - Command::new("rustup") - .stdout(std::process::Stdio::null()) - .args(&["toolchain", "link", "stage1", &stage_path]) - .output() - .map_or(false, |output| output.status.success()) -} - -fn ensure_stage1_toolchain_placeholder_exists(stage_path: &str) -> bool { - let pathbuf = PathBuf::from(stage_path); - - if fs::create_dir_all(pathbuf.join("lib")).is_err() { - return false; - }; - - let pathbuf = pathbuf.join("bin"); - if fs::create_dir_all(&pathbuf).is_err() { - return false; - }; - - let pathbuf = pathbuf.join(format!("rustc{EXE_SUFFIX}")); - - if pathbuf.exists() { - return true; - } - - // Take care not to overwrite the file - let result = File::options().append(true).create(true).open(&pathbuf); - if result.is_err() { - return false; - } - - return true; -} - -// Used to get the path for `Subcommand::Setup` -pub fn interactive_path() -> io::Result { - fn abbrev_all() -> impl Iterator { - ('a'..) - .zip(1..) - .map(|(letter, number)| (letter.to_string(), number.to_string())) - .zip(Profile::all()) - } - - fn parse_with_abbrev(input: &str) -> Result { - let input = input.trim().to_lowercase(); - for ((letter, number), profile) in abbrev_all() { - if input == letter || input == number { - return Ok(profile); - } - } - input.parse() - } - - println!("Welcome to the Rust project! What do you want to do with x.py?"); - for ((letter, _), profile) in abbrev_all() { - println!("{}) {}: {}", letter, profile, profile.purpose()); - } - let template = loop { - print!( - "Please choose one ({}): ", - abbrev_all().map(|((l, _), _)| l).collect::>().join("/") - ); - io::stdout().flush()?; - let mut input = String::new(); - io::stdin().read_line(&mut input)?; - if input.is_empty() { - eprintln!("EOF on stdin, when expecting answer to question. Giving up."); - crate::exit!(1); - } - break match parse_with_abbrev(&input) { - Ok(profile) => profile, - Err(err) => { - eprintln!("error: {err}"); - eprintln!("note: press Ctrl+C to exit"); - continue; - } - }; - }; - Ok(template) -} - -#[derive(PartialEq)] -enum PromptResult { - Yes, // y/Y/yes - No, // n/N/no - Print, // p/P/print -} - -/// Prompt a user for a answer, looping until they enter an accepted input or nothing -fn prompt_user(prompt: &str) -> io::Result> { - let mut input = String::new(); - loop { - print!("{prompt} "); - io::stdout().flush()?; - input.clear(); - io::stdin().read_line(&mut input)?; - match input.trim().to_lowercase().as_str() { - "y" | "yes" => return Ok(Some(PromptResult::Yes)), - "n" | "no" => return Ok(Some(PromptResult::No)), - "p" | "print" => return Ok(Some(PromptResult::Print)), - "" => return Ok(None), - _ => { - eprintln!("error: unrecognized option '{}'", input.trim()); - eprintln!("note: press Ctrl+C to exit"); - } - }; - } -} - -/// Installs `src/etc/pre-push.sh` as a Git hook -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] -pub struct Hook; - -impl Step for Hook { - type Output = (); - const DEFAULT: bool = true; - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("hook") - } - fn make_run(run: RunConfig<'_>) { - if run.builder.config.dry_run() { - return; - } - if let [cmd] = &run.paths[..] { - if cmd.assert_single_path().path.as_path().as_os_str() == "hook" { - run.builder.ensure(Hook); - } - } - } - fn run(self, builder: &Builder<'_>) -> Self::Output { - let config = &builder.config; - if config.dry_run() { - return; - } - t!(install_git_hook_maybe(&config)); - } -} - -// install a git hook to automatically run tidy, if they want -fn install_git_hook_maybe(config: &Config) -> io::Result<()> { - let git = t!(config.git().args(&["rev-parse", "--git-common-dir"]).output().map(|output| { - assert!(output.status.success(), "failed to run `git`"); - PathBuf::from(t!(String::from_utf8(output.stdout)).trim()) - })); - let dst = git.join("hooks").join("pre-push"); - if dst.exists() { - // The git hook has already been set up, or the user already has a custom hook. - return Ok(()); - } - - println!( - "\nRust's CI will automatically fail if it doesn't pass `tidy`, the internal tool for ensuring code quality. -If you'd like, x.py can install a git hook for you that will automatically run `test tidy` before -pushing your code to ensure your code is up to par. If you decide later that this behavior is -undesirable, simply delete the `pre-push` file from .git/hooks." - ); - - if prompt_user("Would you like to install the git hook?: [y/N]")? != Some(PromptResult::Yes) { - println!("Ok, skipping installation!"); - return Ok(()); - } - let src = config.src.join("src").join("etc").join("pre-push.sh"); - match fs::hard_link(src, &dst) { - Err(e) => { - eprintln!( - "error: could not create hook {}: do you already have the git hook installed?\n{}", - dst.display(), - e - ); - return Err(e); - } - Ok(_) => println!("Linked `src/etc/pre-push.sh` to `.git/hooks/pre-push`"), - }; - Ok(()) -} - -/// Sets up or displays `src/etc/rust_analyzer_settings.json` -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] -pub struct Vscode; - -impl Step for Vscode { - type Output = (); - const DEFAULT: bool = true; - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("vscode") - } - fn make_run(run: RunConfig<'_>) { - if run.builder.config.dry_run() { - return; - } - if let [cmd] = &run.paths[..] { - if cmd.assert_single_path().path.as_path().as_os_str() == "vscode" { - run.builder.ensure(Vscode); - } - } - } - fn run(self, builder: &Builder<'_>) -> Self::Output { - let config = &builder.config; - if config.dry_run() { - return; - } - t!(create_vscode_settings_maybe(&config)); - } -} - -/// Create a `.vscode/settings.json` file for rustc development, or just print it -fn create_vscode_settings_maybe(config: &Config) -> io::Result<()> { - let (current_hash, historical_hashes) = SETTINGS_HASHES.split_last().unwrap(); - let vscode_settings = config.src.join(".vscode").join("settings.json"); - // If None, no settings.json exists - // If Some(true), is a previous version of settings.json - // If Some(false), is not a previous version (i.e. user modified) - // If it's up to date we can just skip this - let mut mismatched_settings = None; - if let Ok(current) = fs::read_to_string(&vscode_settings) { - let mut hasher = sha2::Sha256::new(); - hasher.update(¤t); - let hash = hex::encode(hasher.finalize().as_slice()); - if hash == *current_hash { - return Ok(()); - } else if historical_hashes.contains(&hash.as_str()) { - mismatched_settings = Some(true); - } else { - mismatched_settings = Some(false); - } - } - println!( - "\nx.py can automatically install the recommended `.vscode/settings.json` file for rustc development" - ); - match mismatched_settings { - Some(true) => eprintln!( - "warning: existing `.vscode/settings.json` is out of date, x.py will update it" - ), - Some(false) => eprintln!( - "warning: existing `.vscode/settings.json` has been modified by user, x.py will back it up and replace it" - ), - _ => (), - } - let should_create = match prompt_user( - "Would you like to create/update `settings.json`, or only print suggested settings?: [y/p/N]", - )? { - Some(PromptResult::Yes) => true, - Some(PromptResult::Print) => false, - _ => { - println!("Ok, skipping settings!"); - return Ok(()); - } - }; - if should_create { - let path = config.src.join(".vscode"); - if !path.exists() { - fs::create_dir(&path)?; - } - let verb = match mismatched_settings { - // exists but outdated, we can replace this - Some(true) => "Updated", - // exists but user modified, back it up - Some(false) => { - // exists and is not current version or outdated, so back it up - let mut backup = vscode_settings.clone(); - backup.set_extension("json.bak"); - eprintln!("warning: copying `settings.json` to `settings.json.bak`"); - fs::copy(&vscode_settings, &backup)?; - "Updated" - } - _ => "Created", - }; - fs::write(&vscode_settings, &RUST_ANALYZER_SETTINGS)?; - println!("{verb} `.vscode/settings.json`"); - } else { - println!("\n{RUST_ANALYZER_SETTINGS}"); - } - Ok(()) -} diff --git a/src/bootstrap/setup/tests.rs b/src/bootstrap/setup/tests.rs deleted file mode 100644 index 0fe6e4a46..000000000 --- a/src/bootstrap/setup/tests.rs +++ /dev/null @@ -1,14 +0,0 @@ -use super::{RUST_ANALYZER_SETTINGS, SETTINGS_HASHES}; -use sha2::Digest; - -#[test] -fn check_matching_settings_hash() { - let mut hasher = sha2::Sha256::new(); - hasher.update(&RUST_ANALYZER_SETTINGS); - let hash = hex::encode(hasher.finalize().as_slice()); - assert_eq!( - &hash, - SETTINGS_HASHES.last().unwrap(), - "Update `SETTINGS_HASHES` with the new hash of `src/etc/rust_analyzer_settings.json`" - ); -} diff --git a/src/bootstrap/src/bin/main.rs b/src/bootstrap/src/bin/main.rs new file mode 100644 index 000000000..0a6072ae1 --- /dev/null +++ b/src/bootstrap/src/bin/main.rs @@ -0,0 +1,143 @@ +//! rustbuild, the Rust build system +//! +//! This is the entry point for the build system used to compile the `rustc` +//! compiler. Lots of documentation can be found in the `README.md` file in the +//! parent directory, and otherwise documentation can be found throughout the `build` +//! directory in each respective module. + +#[cfg(all(any(unix, windows), not(target_os = "solaris")))] +use std::io::Write; +#[cfg(all(any(unix, windows), not(target_os = "solaris")))] +use std::process; +use std::{env, fs}; + +#[cfg(all(any(unix, windows), not(target_os = "solaris")))] +use bootstrap::t; +use bootstrap::{find_recent_config_change_ids, Build, Config, Subcommand, CONFIG_CHANGE_HISTORY}; + +fn main() { + let args = env::args().skip(1).collect::>(); + let config = Config::parse(&args); + + #[cfg(all(any(unix, windows), not(target_os = "solaris")))] + let mut build_lock; + #[cfg(all(any(unix, windows), not(target_os = "solaris")))] + let _build_lock_guard; + #[cfg(all(any(unix, windows), not(target_os = "solaris")))] + // Display PID of process holding the lock + // PID will be stored in a lock file + { + let path = config.out.join("lock"); + let pid = match fs::read_to_string(&path) { + Ok(contents) => contents, + Err(_) => String::new(), + }; + + build_lock = + fd_lock::RwLock::new(t!(fs::OpenOptions::new().write(true).create(true).open(&path))); + _build_lock_guard = match build_lock.try_write() { + Ok(mut lock) => { + t!(lock.write(&process::id().to_string().as_ref())); + lock + } + err => { + drop(err); + println!("WARNING: build directory locked by process {pid}, waiting for lock"); + let mut lock = t!(build_lock.write()); + t!(lock.write(&process::id().to_string().as_ref())); + lock + } + }; + } + + #[cfg(any(not(any(unix, windows)), target_os = "solaris"))] + println!("WARNING: file locking not supported for target, not locking build directory"); + + // check_version warnings are not printed during setup + let changelog_suggestion = + if matches!(config.cmd, Subcommand::Setup { .. }) { None } else { check_version(&config) }; + + // NOTE: Since `./configure` generates a `config.toml`, distro maintainers will see the + // changelog warning, not the `x.py setup` message. + let suggest_setup = config.config.is_none() && !matches!(config.cmd, Subcommand::Setup { .. }); + if suggest_setup { + println!("WARNING: you have not made a `config.toml`"); + println!( + "HELP: consider running `./x.py setup` or copying `config.example.toml` by running \ + `cp config.example.toml config.toml`" + ); + } else if let Some(suggestion) = &changelog_suggestion { + println!("{suggestion}"); + } + + let pre_commit = config.src.join(".git").join("hooks").join("pre-commit"); + Build::new(config).build(); + + if suggest_setup { + println!("WARNING: you have not made a `config.toml`"); + println!( + "HELP: consider running `./x.py setup` or copying `config.example.toml` by running \ + `cp config.example.toml config.toml`" + ); + } else if let Some(suggestion) = &changelog_suggestion { + println!("{suggestion}"); + } + + // Give a warning if the pre-commit script is in pre-commit and not pre-push. + // HACK: Since the commit script uses hard links, we can't actually tell if it was installed by x.py setup or not. + // We could see if it's identical to src/etc/pre-push.sh, but pre-push may have been modified in the meantime. + // Instead, look for this comment, which is almost certainly not in any custom hook. + if fs::read_to_string(pre_commit).map_or(false, |contents| { + contents.contains("https://github.com/rust-lang/rust/issues/77620#issuecomment-705144570") + }) { + println!( + "WARNING: You have the pre-push script installed to .git/hooks/pre-commit. \ + Consider moving it to .git/hooks/pre-push instead, which runs less often." + ); + } + + if suggest_setup || changelog_suggestion.is_some() { + println!("NOTE: this message was printed twice to make it more likely to be seen"); + } +} + +fn check_version(config: &Config) -> Option { + let mut msg = String::new(); + + if config.changelog_seen.is_some() { + msg.push_str("WARNING: The use of `changelog-seen` is deprecated. Please refer to `change-id` option in `config.example.toml` instead.\n"); + } + + let latest_config_id = CONFIG_CHANGE_HISTORY.last().unwrap(); + if let Some(id) = config.change_id { + if &id == latest_config_id { + return None; + } + + let change_links: Vec = find_recent_config_change_ids(id) + .iter() + .map(|id| format!("https://github.com/rust-lang/rust/pull/{id}")) + .collect(); + if !change_links.is_empty() { + msg.push_str("WARNING: there have been changes to x.py since you last updated.\n"); + msg.push_str("To see more detail about these changes, visit the following PRs:\n"); + + for link in change_links { + msg.push_str(&format!(" - {link}\n")); + } + + msg.push_str("WARNING: there have been changes to x.py since you last updated.\n"); + + msg.push_str("NOTE: to silence this warning, "); + msg.push_str(&format!( + "update `config.toml` to use `change-id = {latest_config_id}` instead" + )); + } + } else { + msg.push_str("WARNING: The `change-id` is missing in the `config.toml`. This means that you will not be able to track the major changes made to the bootstrap configurations.\n"); + msg.push_str("NOTE: to silence this warning, "); + msg.push_str(&format!("add `change-id = {latest_config_id}` at the top of `config.toml`")); + }; + + Some(msg) +} diff --git a/src/bootstrap/src/bin/rustc.rs b/src/bootstrap/src/bin/rustc.rs new file mode 100644 index 000000000..070a2da6a --- /dev/null +++ b/src/bootstrap/src/bin/rustc.rs @@ -0,0 +1,406 @@ +//! Shim which is passed to Cargo as "rustc" when running the bootstrap. +//! +//! This shim will take care of some various tasks that our build process +//! requires that Cargo can't quite do through normal configuration: +//! +//! 1. When compiling build scripts and build dependencies, we need a guaranteed +//! full standard library available. The only compiler which actually has +//! this is the snapshot, so we detect this situation and always compile with +//! the snapshot compiler. +//! 2. We pass a bunch of `--cfg` and other flags based on what we're compiling +//! (and this slightly differs based on a whether we're using a snapshot or +//! not), so we do that all here. +//! +//! This may one day be replaced by RUSTFLAGS, but the dynamic nature of +//! switching compilers for the bootstrap and for build scripts will probably +//! never get replaced. + +use std::env; +use std::path::PathBuf; +use std::process::{Child, Command}; +use std::time::Instant; + +use dylib_util::{dylib_path, dylib_path_var}; + +#[path = "../utils/bin_helpers.rs"] +mod bin_helpers; + +#[path = "../utils/dylib.rs"] +mod dylib_util; + +fn main() { + let args = env::args_os().skip(1).collect::>(); + let arg = |name| args.windows(2).find(|args| args[0] == name).and_then(|args| args[1].to_str()); + + // We don't use the stage in this shim, but let's parse it to make sure that we're invoked + // by bootstrap, or that we provide a helpful error message if not. + bin_helpers::parse_rustc_stage(); + let verbose = bin_helpers::parse_rustc_verbose(); + + // Detect whether or not we're a build script depending on whether --target + // is passed (a bit janky...) + let target = arg("--target"); + let version = args.iter().find(|w| &**w == "-vV"); + + // Use a different compiler for build scripts, since there may not yet be a + // libstd for the real compiler to use. However, if Cargo is attempting to + // determine the version of the compiler, the real compiler needs to be + // used. Currently, these two states are differentiated based on whether + // --target and -vV is/isn't passed. + let (rustc, libdir) = if target.is_none() && version.is_none() { + ("RUSTC_SNAPSHOT", "RUSTC_SNAPSHOT_LIBDIR") + } else { + ("RUSTC_REAL", "RUSTC_LIBDIR") + }; + + let sysroot = env::var_os("RUSTC_SYSROOT").expect("RUSTC_SYSROOT was not set"); + let on_fail = env::var_os("RUSTC_ON_FAIL").map(Command::new); + + let rustc = env::var_os(rustc).unwrap_or_else(|| panic!("{:?} was not set", rustc)); + let libdir = env::var_os(libdir).unwrap_or_else(|| panic!("{:?} was not set", libdir)); + let mut dylib_path = dylib_path(); + dylib_path.insert(0, PathBuf::from(&libdir)); + + let mut cmd = Command::new(rustc); + cmd.args(&args).env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); + + // Get the name of the crate we're compiling, if any. + let crate_name = arg("--crate-name"); + + if let Some(crate_name) = crate_name { + if let Some(target) = env::var_os("RUSTC_TIME") { + if target == "all" + || target.into_string().unwrap().split(',').any(|c| c.trim() == crate_name) + { + cmd.arg("-Ztime-passes"); + } + } + } + + // Print backtrace in case of ICE + if env::var("RUSTC_BACKTRACE_ON_ICE").is_ok() && env::var("RUST_BACKTRACE").is_err() { + cmd.env("RUST_BACKTRACE", "1"); + } + + if let Ok(lint_flags) = env::var("RUSTC_LINT_FLAGS") { + cmd.args(lint_flags.split_whitespace()); + } + + if target.is_some() { + // The stage0 compiler has a special sysroot distinct from what we + // actually downloaded, so we just always pass the `--sysroot` option, + // unless one is already set. + if !args.iter().any(|arg| arg == "--sysroot") { + cmd.arg("--sysroot").arg(&sysroot); + } + + // If we're compiling specifically the `panic_abort` crate then we pass + // the `-C panic=abort` option. Note that we do not do this for any + // other crate intentionally as this is the only crate for now that we + // ship with panic=abort. + // + // This... is a bit of a hack how we detect this. Ideally this + // information should be encoded in the crate I guess? Would likely + // require an RFC amendment to RFC 1513, however. + if crate_name == Some("panic_abort") { + cmd.arg("-C").arg("panic=abort"); + } + + // `-Ztls-model=initial-exec` must not be applied to proc-macros, see + // issue https://github.com/rust-lang/rust/issues/100530 + if env::var("RUSTC_TLS_MODEL_INITIAL_EXEC").is_ok() + && arg("--crate-type") != Some("proc-macro") + && !matches!(crate_name, Some("proc_macro2" | "quote" | "syn" | "synstructure")) + { + cmd.arg("-Ztls-model=initial-exec"); + } + } else { + // Find any host flags that were passed by bootstrap. + // The flags are stored in a RUSTC_HOST_FLAGS variable, separated by spaces. + if let Ok(flags) = std::env::var("RUSTC_HOST_FLAGS") { + for flag in flags.split(' ') { + cmd.arg(flag); + } + } + } + + if let Ok(map) = env::var("RUSTC_DEBUGINFO_MAP") { + cmd.arg("--remap-path-prefix").arg(&map); + } + // The remap flags for Cargo registry sources need to be passed after the remapping for the + // Rust source code directory, to handle cases when $CARGO_HOME is inside the source directory. + if let Ok(maps) = env::var("RUSTC_CARGO_REGISTRY_SRC_TO_REMAP") { + for map in maps.split('\t') { + cmd.arg("--remap-path-prefix").arg(map); + } + } + + // Force all crates compiled by this compiler to (a) be unstable and (b) + // allow the `rustc_private` feature to link to other unstable crates + // also in the sysroot. We also do this for host crates, since those + // may be proc macros, in which case we might ship them. + if env::var_os("RUSTC_FORCE_UNSTABLE").is_some() { + cmd.arg("-Z").arg("force-unstable-if-unmarked"); + } + + // allow-features is handled from within this rustc wrapper because of + // issues with build scripts. Some packages use build scripts to + // dynamically detect if certain nightly features are available. + // There are different ways this causes problems: + // + // * rustix runs `rustc` on a small test program to see if the feature is + // available (and sets a `cfg` if it is). It does not honor + // CARGO_ENCODED_RUSTFLAGS. + // * proc-macro2 detects if `rustc -vV` says "nighty" or "dev" and enables + // nightly features. It will scan CARGO_ENCODED_RUSTFLAGS for + // -Zallow-features. Unfortunately CARGO_ENCODED_RUSTFLAGS is not set + // for build-dependencies when --target is used. + // + // The issues above means we can't just use RUSTFLAGS, and we can't use + // `cargo -Zallow-features=…`. Passing it through here ensures that it + // always gets set. Unfortunately that also means we need to enable more + // features than we really want (like those for proc-macro2), but there + // isn't much of a way around it. + // + // I think it is unfortunate that build scripts are doing this at all, + // since changes to nightly features can cause crates to break even if the + // user didn't want or care about the use of the nightly features. I think + // nightly features should be opt-in only. Unfortunately the dynamic + // checks are now too wide spread that we just need to deal with it. + // + // If you want to try to remove this, I suggest working with the crate + // authors to remove the dynamic checking. Another option is to pursue + // https://github.com/rust-lang/cargo/issues/11244 and + // https://github.com/rust-lang/cargo/issues/4423, which will likely be + // very difficult, but could help expose -Zallow-features into build + // scripts so they could try to honor them. + if let Ok(allow_features) = env::var("RUSTC_ALLOW_FEATURES") { + cmd.arg(format!("-Zallow-features={allow_features}")); + } + + if let Ok(flags) = env::var("MAGIC_EXTRA_RUSTFLAGS") { + for flag in flags.split(' ') { + cmd.arg(flag); + } + } + + let is_test = args.iter().any(|a| a == "--test"); + if verbose > 2 { + let rust_env_vars = + env::vars().filter(|(k, _)| k.starts_with("RUST") || k.starts_with("CARGO")); + let prefix = if is_test { "[RUSTC-SHIM] rustc --test" } else { "[RUSTC-SHIM] rustc" }; + let prefix = match crate_name { + Some(crate_name) => format!("{prefix} {crate_name}"), + None => prefix.to_string(), + }; + for (i, (k, v)) in rust_env_vars.enumerate() { + eprintln!("{prefix} env[{i}]: {k:?}={v:?}"); + } + eprintln!("{} working directory: {}", prefix, env::current_dir().unwrap().display()); + eprintln!( + "{} command: {:?}={:?} {:?}", + prefix, + dylib_path_var(), + env::join_paths(&dylib_path).unwrap(), + cmd, + ); + eprintln!("{prefix} sysroot: {sysroot:?}"); + eprintln!("{prefix} libdir: {libdir:?}"); + } + + if env::var_os("RUSTC_BOLT_LINK_FLAGS").is_some() { + if let Some("rustc_driver") = crate_name { + cmd.arg("-Clink-args=-Wl,-q"); + } + } + + let start = Instant::now(); + let (child, status) = { + let errmsg = format!("\nFailed to run:\n{cmd:?}\n-------------"); + let mut child = cmd.spawn().expect(&errmsg); + let status = child.wait().expect(&errmsg); + (child, status) + }; + + if env::var_os("RUSTC_PRINT_STEP_TIMINGS").is_some() + || env::var_os("RUSTC_PRINT_STEP_RUSAGE").is_some() + { + if let Some(crate_name) = crate_name { + let dur = start.elapsed(); + // If the user requested resource usage data, then + // include that in addition to the timing output. + let rusage_data = + env::var_os("RUSTC_PRINT_STEP_RUSAGE").and_then(|_| format_rusage_data(child)); + eprintln!( + "[RUSTC-TIMING] {} test:{} {}.{:03}{}{}", + crate_name, + is_test, + dur.as_secs(), + dur.subsec_millis(), + if rusage_data.is_some() { " " } else { "" }, + rusage_data.unwrap_or(String::new()), + ); + } + } + + if status.success() { + std::process::exit(0); + // NOTE: everything below here is unreachable. do not put code that + // should run on success, after this block. + } + if verbose > 0 { + println!("\nDid not run successfully: {status}\n{cmd:?}\n-------------"); + } + + if let Some(mut on_fail) = on_fail { + on_fail.status().expect("Could not run the on_fail command"); + } + + // Preserve the exit code. In case of signal, exit with 0xfe since it's + // awkward to preserve this status in a cross-platform way. + match status.code() { + Some(i) => std::process::exit(i), + None => { + eprintln!("rustc exited with {status}"); + std::process::exit(0xfe); + } + } +} + +#[cfg(all(not(unix), not(windows)))] +// In the future we can add this for more platforms +fn format_rusage_data(_child: Child) -> Option { + None +} + +#[cfg(windows)] +fn format_rusage_data(child: Child) -> Option { + use std::os::windows::io::AsRawHandle; + + use windows::{ + Win32::Foundation::HANDLE, + Win32::System::ProcessStatus::{ + K32GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS, PROCESS_MEMORY_COUNTERS_EX, + }, + Win32::System::Threading::GetProcessTimes, + Win32::System::Time::FileTimeToSystemTime, + }; + + let handle = HANDLE(child.as_raw_handle() as isize); + + let mut user_filetime = Default::default(); + let mut user_time = Default::default(); + let mut kernel_filetime = Default::default(); + let mut kernel_time = Default::default(); + let mut memory_counters = PROCESS_MEMORY_COUNTERS::default(); + + unsafe { + GetProcessTimes( + handle, + &mut Default::default(), + &mut Default::default(), + &mut kernel_filetime, + &mut user_filetime, + ) + } + .ok()?; + unsafe { FileTimeToSystemTime(&user_filetime, &mut user_time) }.ok()?; + unsafe { FileTimeToSystemTime(&kernel_filetime, &mut kernel_time) }.ok()?; + + // Unlike on Linux with RUSAGE_CHILDREN, this will only return memory information for the process + // with the given handle and none of that process's children. + unsafe { + K32GetProcessMemoryInfo( + handle, + &mut memory_counters, + std::mem::size_of::() as u32, + ) + } + .ok() + .ok()?; + + // Guide on interpreting these numbers: + // https://docs.microsoft.com/en-us/windows/win32/psapi/process-memory-usage-information + let peak_working_set = memory_counters.PeakWorkingSetSize / 1024; + let peak_page_file = memory_counters.PeakPagefileUsage / 1024; + let peak_paged_pool = memory_counters.QuotaPeakPagedPoolUsage / 1024; + let peak_nonpaged_pool = memory_counters.QuotaPeakNonPagedPoolUsage / 1024; + Some(format!( + "user: {USER_SEC}.{USER_USEC:03} \ + sys: {SYS_SEC}.{SYS_USEC:03} \ + peak working set (kb): {PEAK_WORKING_SET} \ + peak page file usage (kb): {PEAK_PAGE_FILE} \ + peak paged pool usage (kb): {PEAK_PAGED_POOL} \ + peak non-paged pool usage (kb): {PEAK_NONPAGED_POOL} \ + page faults: {PAGE_FAULTS}", + USER_SEC = user_time.wSecond + (user_time.wMinute * 60), + USER_USEC = user_time.wMilliseconds, + SYS_SEC = kernel_time.wSecond + (kernel_time.wMinute * 60), + SYS_USEC = kernel_time.wMilliseconds, + PEAK_WORKING_SET = peak_working_set, + PEAK_PAGE_FILE = peak_page_file, + PEAK_PAGED_POOL = peak_paged_pool, + PEAK_NONPAGED_POOL = peak_nonpaged_pool, + PAGE_FAULTS = memory_counters.PageFaultCount, + )) +} + +#[cfg(unix)] +/// Tries to build a string with human readable data for several of the rusage +/// fields. Note that we are focusing mainly on data that we believe to be +/// supplied on Linux (the `rusage` struct has other fields in it but they are +/// currently unsupported by Linux). +fn format_rusage_data(_child: Child) -> Option { + let rusage: libc::rusage = unsafe { + let mut recv = std::mem::zeroed(); + // -1 is RUSAGE_CHILDREN, which means to get the rusage for all children + // (and grandchildren, etc) processes that have respectively terminated + // and been waited for. + let retval = libc::getrusage(-1, &mut recv); + if retval != 0 { + return None; + } + recv + }; + // Mac OS X reports the maxrss in bytes, not kb. + let divisor = if env::consts::OS == "macos" { 1024 } else { 1 }; + let maxrss = (rusage.ru_maxrss + (divisor - 1)) / divisor; + + let mut init_str = format!( + "user: {USER_SEC}.{USER_USEC:03} \ + sys: {SYS_SEC}.{SYS_USEC:03} \ + max rss (kb): {MAXRSS}", + USER_SEC = rusage.ru_utime.tv_sec, + USER_USEC = rusage.ru_utime.tv_usec, + SYS_SEC = rusage.ru_stime.tv_sec, + SYS_USEC = rusage.ru_stime.tv_usec, + MAXRSS = maxrss + ); + + // The remaining rusage stats vary in platform support. So we treat + // uniformly zero values in each category as "not worth printing", since it + // either means no events of that type occurred, or that the platform + // does not support it. + + let minflt = rusage.ru_minflt; + let majflt = rusage.ru_majflt; + if minflt != 0 || majflt != 0 { + init_str.push_str(&format!(" page reclaims: {minflt} page faults: {majflt}")); + } + + let inblock = rusage.ru_inblock; + let oublock = rusage.ru_oublock; + if inblock != 0 || oublock != 0 { + init_str.push_str(&format!(" fs block inputs: {inblock} fs block outputs: {oublock}")); + } + + let nvcsw = rusage.ru_nvcsw; + let nivcsw = rusage.ru_nivcsw; + if nvcsw != 0 || nivcsw != 0 { + init_str.push_str(&format!( + " voluntary ctxt switches: {nvcsw} involuntary ctxt switches: {nivcsw}" + )); + } + + return Some(init_str); +} diff --git a/src/bootstrap/src/bin/rustdoc.rs b/src/bootstrap/src/bin/rustdoc.rs new file mode 100644 index 000000000..dbbce6fe2 --- /dev/null +++ b/src/bootstrap/src/bin/rustdoc.rs @@ -0,0 +1,92 @@ +//! Shim which is passed to Cargo as "rustdoc" when running the bootstrap. +//! +//! See comments in `src/bootstrap/rustc.rs` for more information. + +use std::env; +use std::ffi::OsString; +use std::path::PathBuf; +use std::process::Command; + +use dylib_util::{dylib_path, dylib_path_var}; + +#[path = "../utils/bin_helpers.rs"] +mod bin_helpers; + +#[path = "../utils/dylib.rs"] +mod dylib_util; + +fn main() { + let args = env::args_os().skip(1).collect::>(); + + let stage = bin_helpers::parse_rustc_stage(); + let verbose = bin_helpers::parse_rustc_verbose(); + + let rustdoc = env::var_os("RUSTDOC_REAL").expect("RUSTDOC_REAL was not set"); + let libdir = env::var_os("RUSTDOC_LIBDIR").expect("RUSTDOC_LIBDIR was not set"); + let sysroot = env::var_os("RUSTC_SYSROOT").expect("RUSTC_SYSROOT was not set"); + + // Detect whether or not we're a build script depending on whether --target + // is passed (a bit janky...) + let target = args.windows(2).find(|w| &*w[0] == "--target").and_then(|w| w[1].to_str()); + + let mut dylib_path = dylib_path(); + dylib_path.insert(0, PathBuf::from(libdir.clone())); + + let mut cmd = Command::new(rustdoc); + + if target.is_some() { + // The stage0 compiler has a special sysroot distinct from what we + // actually downloaded, so we just always pass the `--sysroot` option, + // unless one is already set. + if !args.iter().any(|arg| arg == "--sysroot") { + cmd.arg("--sysroot").arg(&sysroot); + } + } + + cmd.args(&args); + cmd.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); + + // Force all crates compiled by this compiler to (a) be unstable and (b) + // allow the `rustc_private` feature to link to other unstable crates + // also in the sysroot. + if env::var_os("RUSTC_FORCE_UNSTABLE").is_some() { + cmd.arg("-Z").arg("force-unstable-if-unmarked"); + } + if let Some(linker) = env::var_os("RUSTDOC_LINKER") { + let mut arg = OsString::from("-Clinker="); + arg.push(&linker); + cmd.arg(arg); + } + if let Ok(no_threads) = env::var("RUSTDOC_LLD_NO_THREADS") { + cmd.arg("-Clink-arg=-fuse-ld=lld"); + cmd.arg(format!("-Clink-arg=-Wl,{no_threads}")); + } + // Cargo doesn't pass RUSTDOCFLAGS to proc_macros: + // https://github.com/rust-lang/cargo/issues/4423 + // Thus, if we are on stage 0, we explicitly set `--cfg=bootstrap`. + // We also declare that the flag is expected, which we need to do to not + // get warnings about it being unexpected. + if stage == "0" { + cmd.arg("--cfg=bootstrap"); + } + cmd.arg("-Zunstable-options"); + // #[cfg(bootstrap)] + cmd.arg("--check-cfg=values(bootstrap)"); + // cmd.arg("--check-cfg=cfg(bootstrap)"); + + if verbose > 1 { + eprintln!( + "rustdoc command: {:?}={:?} {:?}", + dylib_path_var(), + env::join_paths(&dylib_path).unwrap(), + cmd, + ); + eprintln!("sysroot: {sysroot:?}"); + eprintln!("libdir: {libdir:?}"); + } + + std::process::exit(match cmd.status() { + Ok(s) => s.code().unwrap_or(1), + Err(e) => panic!("\n\nfailed to run {cmd:?}: {e}\n\n"), + }) +} diff --git a/src/bootstrap/src/bin/sccache-plus-cl.rs b/src/bootstrap/src/bin/sccache-plus-cl.rs new file mode 100644 index 000000000..554c2dd4d --- /dev/null +++ b/src/bootstrap/src/bin/sccache-plus-cl.rs @@ -0,0 +1,38 @@ +use std::env; +use std::process::{self, Command}; + +fn main() { + let target = env::var("SCCACHE_TARGET").unwrap(); + // Locate the actual compiler that we're invoking + env::set_var("CC", env::var_os("SCCACHE_CC").unwrap()); + env::set_var("CXX", env::var_os("SCCACHE_CXX").unwrap()); + let mut cfg = cc::Build::new(); + cfg.cargo_metadata(false) + .out_dir("/") + .target(&target) + .host(&target) + .opt_level(0) + .warnings(false) + .debug(false); + let compiler = cfg.get_compiler(); + + // Invoke sccache with said compiler + let sccache_path = env::var_os("SCCACHE_PATH").unwrap(); + let mut cmd = Command::new(&sccache_path); + cmd.arg(compiler.path()); + for &(ref k, ref v) in compiler.env() { + cmd.env(k, v); + } + for arg in env::args().skip(1) { + cmd.arg(arg); + } + + if let Ok(s) = env::var("SCCACHE_EXTRA_ARGS") { + for s in s.split_whitespace() { + cmd.arg(s); + } + } + + let status = cmd.status().expect("failed to spawn"); + process::exit(status.code().unwrap_or(2)) +} diff --git a/src/bootstrap/src/core/build_steps/check.rs b/src/bootstrap/src/core/build_steps/check.rs new file mode 100644 index 000000000..121925b56 --- /dev/null +++ b/src/bootstrap/src/core/build_steps/check.rs @@ -0,0 +1,544 @@ +//! Implementation of compiling the compiler and standard library, in "check"-based modes. + +use crate::core::build_steps::compile::{ + add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo, +}; +use crate::core::build_steps::tool::{prepare_tool_cargo, SourceType}; +use crate::core::builder::{crate_description, Alias, Builder, Kind, RunConfig, ShouldRun, Step}; +use crate::core::config::TargetSelection; +use crate::utils::cache::Interned; +use crate::INTERNER; +use crate::{Compiler, Mode, Subcommand}; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Std { + pub target: TargetSelection, + /// Whether to build only a subset of crates. + /// + /// This shouldn't be used from other steps; see the comment on [`compile::Rustc`]. + /// + /// [`compile::Rustc`]: crate::core::build_steps::compile::Rustc + crates: Interned>, +} + +/// Returns args for the subcommand itself (not for cargo) +fn args(builder: &Builder<'_>) -> Vec { + fn strings<'a>(arr: &'a [&str]) -> impl Iterator + 'a { + arr.iter().copied().map(String::from) + } + + if let Subcommand::Clippy { fix, allow, deny, warn, forbid, .. } = &builder.config.cmd { + // disable the most spammy clippy lints + let ignored_lints = vec![ + "many_single_char_names", // there are a lot in stdarch + "collapsible_if", + "type_complexity", + "missing_safety_doc", // almost 3K warnings + "too_many_arguments", + "needless_lifetimes", // people want to keep the lifetimes + "wrong_self_convention", + ]; + let mut args = vec![]; + if *fix { + #[rustfmt::skip] + args.extend(strings(&[ + "--fix", "-Zunstable-options", + // FIXME: currently, `--fix` gives an error while checking tests for libtest, + // possibly because libtest is not yet built in the sysroot. + // As a workaround, avoid checking tests and benches when passed --fix. + "--lib", "--bins", "--examples", + ])); + } + args.extend(strings(&["--", "--cap-lints", "warn"])); + args.extend(ignored_lints.iter().map(|lint| format!("-Aclippy::{}", lint))); + let mut clippy_lint_levels: Vec = Vec::new(); + allow.iter().for_each(|v| clippy_lint_levels.push(format!("-A{}", v))); + deny.iter().for_each(|v| clippy_lint_levels.push(format!("-D{}", v))); + warn.iter().for_each(|v| clippy_lint_levels.push(format!("-W{}", v))); + forbid.iter().for_each(|v| clippy_lint_levels.push(format!("-F{}", v))); + args.extend(clippy_lint_levels); + args.extend(builder.config.free_args.clone()); + args + } else { + builder.config.free_args.clone() + } +} + +fn cargo_subcommand(kind: Kind) -> &'static str { + match kind { + Kind::Check => "check", + Kind::Clippy => "clippy", + Kind::Fix => "fix", + _ => unreachable!(), + } +} + +impl Std { + pub fn new(target: TargetSelection) -> Self { + Self { target, crates: INTERNER.intern_list(vec![]) } + } +} + +impl Step for Std { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.crate_or_deps("sysroot").path("library") + } + + fn make_run(run: RunConfig<'_>) { + let crates = run.make_run_crates(Alias::Library); + run.builder.ensure(Std { target: run.target, crates }); + } + + fn run(self, builder: &Builder<'_>) { + builder.update_submodule(&Path::new("library").join("stdarch")); + + let target = self.target; + let compiler = builder.compiler(builder.top_stage, builder.config.build); + + let mut cargo = builder.cargo( + compiler, + Mode::Std, + SourceType::InTree, + target, + cargo_subcommand(builder.kind), + ); + std_cargo(builder, target, compiler.stage, &mut cargo); + if matches!(builder.config.cmd, Subcommand::Fix { .. }) { + // By default, cargo tries to fix all targets. Tell it not to fix tests until we've added `test` to the sysroot. + cargo.arg("--lib"); + } + + for krate in &*self.crates { + cargo.arg("-p").arg(krate); + } + + let _guard = builder.msg_check( + format_args!("library artifacts{}", crate_description(&self.crates)), + target, + ); + run_cargo( + builder, + cargo, + args(builder), + &libstd_stamp(builder, compiler, target), + vec![], + true, + false, + ); + + // We skip populating the sysroot in non-zero stage because that'll lead + // to rlib/rmeta conflicts if std gets built during this session. + if compiler.stage == 0 { + let libdir = builder.sysroot_libdir(compiler, target); + let hostdir = builder.sysroot_libdir(compiler, compiler.host); + add_to_sysroot(&builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target)); + } + drop(_guard); + + // don't run on std twice with x.py clippy + // don't check test dependencies if we haven't built libtest + if builder.kind == Kind::Clippy || !self.crates.iter().any(|krate| krate == "test") { + return; + } + + // Then run cargo again, once we've put the rmeta files for the library + // crates into the sysroot. This is needed because e.g., core's tests + // depend on `libtest` -- Cargo presumes it will exist, but it doesn't + // since we initialize with an empty sysroot. + // + // Currently only the "libtest" tree of crates does this. + let mut cargo = builder.cargo( + compiler, + Mode::Std, + SourceType::InTree, + target, + cargo_subcommand(builder.kind), + ); + + // If we're not in stage 0, tests and examples will fail to compile + // from `core` definitions being loaded from two different `libcore` + // .rmeta and .rlib files. + if compiler.stage == 0 { + cargo.arg("--all-targets"); + } + + std_cargo(builder, target, compiler.stage, &mut cargo); + + // Explicitly pass -p for all dependencies krates -- this will force cargo + // to also check the tests/benches/examples for these crates, rather + // than just the leaf crate. + for krate in &*self.crates { + cargo.arg("-p").arg(krate); + } + + let _guard = builder.msg_check("library test/bench/example targets", target); + run_cargo( + builder, + cargo, + args(builder), + &libstd_test_stamp(builder, compiler, target), + vec![], + true, + false, + ); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Rustc { + pub target: TargetSelection, + /// Whether to build only a subset of crates. + /// + /// This shouldn't be used from other steps; see the comment on [`compile::Rustc`]. + /// + /// [`compile::Rustc`]: crate::core::build_steps::compile::Rustc + crates: Interned>, +} + +impl Rustc { + pub fn new(target: TargetSelection, builder: &Builder<'_>) -> Self { + let crates = builder + .in_tree_crates("rustc-main", Some(target)) + .into_iter() + .map(|krate| krate.name.to_string()) + .collect(); + Self { target, crates: INTERNER.intern_list(crates) } + } +} + +impl Step for Rustc { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.crate_or_deps("rustc-main").path("compiler") + } + + fn make_run(run: RunConfig<'_>) { + let crates = run.make_run_crates(Alias::Compiler); + run.builder.ensure(Rustc { target: run.target, crates }); + } + + /// Builds the compiler. + /// + /// This will build the compiler for a particular stage of the build using + /// the `compiler` targeting the `target` architecture. The artifacts + /// created will also be linked into the sysroot directory. + fn run(self, builder: &Builder<'_>) { + let compiler = builder.compiler(builder.top_stage, builder.config.build); + let target = self.target; + + if compiler.stage != 0 { + // If we're not in stage 0, then we won't have a std from the beta + // compiler around. That means we need to make sure there's one in + // the sysroot for the compiler to find. Otherwise, we're going to + // fail when building crates that need to generate code (e.g., build + // scripts and their dependencies). + builder.ensure(crate::core::build_steps::compile::Std::new(compiler, compiler.host)); + builder.ensure(crate::core::build_steps::compile::Std::new(compiler, target)); + } else { + builder.ensure(Std::new(target)); + } + + let mut cargo = builder.cargo( + compiler, + Mode::Rustc, + SourceType::InTree, + target, + cargo_subcommand(builder.kind), + ); + rustc_cargo(builder, &mut cargo, target, compiler.stage); + + // For ./x.py clippy, don't run with --all-targets because + // linting tests and benchmarks can produce very noisy results + if builder.kind != Kind::Clippy { + cargo.arg("--all-targets"); + } + + // Explicitly pass -p for all compiler crates -- this will force cargo + // to also check the tests/benches/examples for these crates, rather + // than just the leaf crate. + for krate in &*self.crates { + cargo.arg("-p").arg(krate); + } + + let _guard = builder.msg_check( + format_args!("compiler artifacts{}", crate_description(&self.crates)), + target, + ); + run_cargo( + builder, + cargo, + args(builder), + &librustc_stamp(builder, compiler, target), + vec![], + true, + false, + ); + + let libdir = builder.sysroot_libdir(compiler, target); + let hostdir = builder.sysroot_libdir(compiler, compiler.host); + add_to_sysroot(&builder, &libdir, &hostdir, &librustc_stamp(builder, compiler, target)); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct CodegenBackend { + pub target: TargetSelection, + pub backend: Interned, +} + +impl Step for CodegenBackend { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.paths(&["compiler/rustc_codegen_cranelift", "compiler/rustc_codegen_gcc"]) + } + + fn make_run(run: RunConfig<'_>) { + for &backend in &[INTERNER.intern_str("cranelift"), INTERNER.intern_str("gcc")] { + run.builder.ensure(CodegenBackend { target: run.target, backend }); + } + } + + fn run(self, builder: &Builder<'_>) { + // FIXME: remove once https://github.com/rust-lang/rust/issues/112393 is resolved + if builder.build.config.vendor && &self.backend == "gcc" { + println!("Skipping checking of `rustc_codegen_gcc` with vendoring enabled."); + return; + } + + let compiler = builder.compiler(builder.top_stage, builder.config.build); + let target = self.target; + let backend = self.backend; + + builder.ensure(Rustc::new(target, builder)); + + let mut cargo = builder.cargo( + compiler, + Mode::Codegen, + SourceType::InTree, + target, + cargo_subcommand(builder.kind), + ); + cargo + .arg("--manifest-path") + .arg(builder.src.join(format!("compiler/rustc_codegen_{backend}/Cargo.toml"))); + rustc_cargo_env(builder, &mut cargo, target, compiler.stage); + + let _guard = builder.msg_check(&backend, target); + + run_cargo( + builder, + cargo, + args(builder), + &codegen_backend_stamp(builder, compiler, target, backend), + vec![], + true, + false, + ); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct RustAnalyzer { + pub target: TargetSelection, +} + +impl Step for RustAnalyzer { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/tools/rust-analyzer").default_condition( + builder + .config + .tools + .as_ref() + .map_or(true, |tools| tools.iter().any(|tool| tool == "rust-analyzer")), + ) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustAnalyzer { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let compiler = builder.compiler(builder.top_stage, builder.config.build); + let target = self.target; + + builder.ensure(Std::new(target)); + + let mut cargo = prepare_tool_cargo( + builder, + compiler, + Mode::ToolStd, + target, + cargo_subcommand(builder.kind), + "src/tools/rust-analyzer", + SourceType::InTree, + &["rust-analyzer/in-rust-tree".to_owned()], + ); + + cargo.allow_features(crate::core::build_steps::tool::RustAnalyzer::ALLOW_FEATURES); + + // For ./x.py clippy, don't check those targets because + // linting tests and benchmarks can produce very noisy results + if builder.kind != Kind::Clippy { + // can't use `--all-targets` because `--examples` doesn't work well + cargo.arg("--bins"); + cargo.arg("--tests"); + cargo.arg("--benches"); + } + + let _guard = builder.msg_check("rust-analyzer artifacts", target); + run_cargo( + builder, + cargo, + args(builder), + &stamp(builder, compiler, target), + vec![], + true, + false, + ); + + /// Cargo's output path in a given stage, compiled by a particular + /// compiler for the specified target. + fn stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { + builder.cargo_out(compiler, Mode::ToolStd, target).join(".rust-analyzer-check.stamp") + } + } +} + +macro_rules! tool_check_step { + ($name:ident, $path:literal, $($alias:literal, )* $source_type:path $(, $default:literal )?) => { + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] + pub struct $name { + pub target: TargetSelection, + } + + impl Step for $name { + type Output = (); + const ONLY_HOSTS: bool = true; + // don't ever check out-of-tree tools by default, they'll fail when toolstate is broken + const DEFAULT: bool = matches!($source_type, SourceType::InTree) $( && $default )?; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.paths(&[ $path, $($alias),* ]) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure($name { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let compiler = builder.compiler(builder.top_stage, builder.config.build); + let target = self.target; + + builder.ensure(Rustc::new(target, builder)); + + let mut cargo = prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + target, + cargo_subcommand(builder.kind), + $path, + $source_type, + &[], + ); + + // For ./x.py clippy, don't run with --all-targets because + // linting tests and benchmarks can produce very noisy results + if builder.kind != Kind::Clippy { + cargo.arg("--all-targets"); + } + + // Enable internal lints for clippy and rustdoc + // NOTE: this doesn't enable lints for any other tools unless they explicitly add `#![warn(rustc::internal)]` + // See https://github.com/rust-lang/rust/pull/80573#issuecomment-754010776 + cargo.rustflag("-Zunstable-options"); + let _guard = builder.msg_check(&concat!(stringify!($name), " artifacts").to_lowercase(), target); + run_cargo( + builder, + cargo, + args(builder), + &stamp(builder, compiler, target), + vec![], + true, + false, + ); + + /// Cargo's output path in a given stage, compiled by a particular + /// compiler for the specified target. + fn stamp( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, + ) -> PathBuf { + builder + .cargo_out(compiler, Mode::ToolRustc, target) + .join(format!(".{}-check.stamp", stringify!($name).to_lowercase())) + } + } + } + }; +} + +tool_check_step!(Rustdoc, "src/tools/rustdoc", "src/librustdoc", SourceType::InTree); +// Clippy, miri and Rustfmt are hybrids. They are external tools, but use a git subtree instead +// of a submodule. Since the SourceType only drives the deny-warnings +// behavior, treat it as in-tree so that any new warnings in clippy will be +// rejected. +tool_check_step!(Clippy, "src/tools/clippy", SourceType::InTree); +tool_check_step!(Miri, "src/tools/miri", SourceType::InTree); +tool_check_step!(CargoMiri, "src/tools/miri/cargo-miri", SourceType::InTree); +tool_check_step!(Rls, "src/tools/rls", SourceType::InTree); +tool_check_step!(Rustfmt, "src/tools/rustfmt", SourceType::InTree); +tool_check_step!(MiroptTestTools, "src/tools/miropt-test-tools", SourceType::InTree); + +tool_check_step!(Bootstrap, "src/bootstrap", SourceType::InTree, false); + +/// Cargo's output path for the standard library in a given stage, compiled +/// by a particular compiler for the specified target. +fn libstd_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { + builder.cargo_out(compiler, Mode::Std, target).join(".libstd-check.stamp") +} + +/// Cargo's output path for the standard library in a given stage, compiled +/// by a particular compiler for the specified target. +fn libstd_test_stamp( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, +) -> PathBuf { + builder.cargo_out(compiler, Mode::Std, target).join(".libstd-check-test.stamp") +} + +/// Cargo's output path for librustc in a given stage, compiled by a particular +/// compiler for the specified target. +fn librustc_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { + builder.cargo_out(compiler, Mode::Rustc, target).join(".librustc-check.stamp") +} + +/// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular +/// compiler for the specified target and backend. +fn codegen_backend_stamp( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, + backend: Interned, +) -> PathBuf { + builder + .cargo_out(compiler, Mode::Codegen, target) + .join(format!(".librustc_codegen_{backend}-check.stamp")) +} diff --git a/src/bootstrap/src/core/build_steps/clean.rs b/src/bootstrap/src/core/build_steps/clean.rs new file mode 100644 index 000000000..cbb6b5f46 --- /dev/null +++ b/src/bootstrap/src/core/build_steps/clean.rs @@ -0,0 +1,242 @@ +//! Implementation of `make clean` in rustbuild. +//! +//! Responsible for cleaning out a build directory of all old and stale +//! artifacts to prepare for a fresh build. Currently doesn't remove the +//! `build/cache` directory (download cache) or the `build/$target/llvm` +//! directory unless the `--all` flag is present. + +use std::fs; +use std::io::{self, ErrorKind}; +use std::path::Path; + +use crate::core::builder::{crate_description, Builder, RunConfig, ShouldRun, Step}; +use crate::utils::cache::Interned; +use crate::utils::helpers::t; +use crate::{Build, Compiler, Mode, Subcommand}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct CleanAll {} + +impl Step for CleanAll { + const DEFAULT: bool = true; + type Output = (); + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(CleanAll {}) + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + let Subcommand::Clean { all, stage } = builder.config.cmd else { + unreachable!("wrong subcommand?") + }; + + if all && stage.is_some() { + panic!("--all and --stage can't be used at the same time for `x clean`"); + } + + clean(builder.build, all, stage) + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() // handled by DEFAULT + } +} + +macro_rules! clean_crate_tree { + ( $( $name:ident, $mode:path, $root_crate:literal);+ $(;)? ) => { $( + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + pub struct $name { + compiler: Compiler, + crates: Interned>, + } + + impl Step for $name { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let crates = run.builder.in_tree_crates($root_crate, None); + run.crates(crates) + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + let compiler = builder.compiler(builder.top_stage, run.target); + builder.ensure(Self { crates: run.cargo_crates_in_set(), compiler }); + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + let compiler = self.compiler; + let target = compiler.host; + let mut cargo = builder.bare_cargo(compiler, $mode, target, "clean"); + + // Since https://github.com/rust-lang/rust/pull/111076 enables + // unstable cargo feature (`public-dependency`), we need to ensure + // that unstable features are enabled before reading libstd Cargo.toml. + cargo.env("RUSTC_BOOTSTRAP", "1"); + + for krate in &*self.crates { + cargo.arg("-p"); + cargo.arg(krate); + } + + builder.info(&format!( + "Cleaning{} stage{} {} artifacts ({} -> {})", + crate_description(&self.crates), compiler.stage, stringify!($name).to_lowercase(), &compiler.host, target, + )); + + // NOTE: doesn't use `run_cargo` because we don't want to save a stamp file, + // and doesn't use `stream_cargo` to avoid passing `--message-format` which `clean` doesn't accept. + builder.run(&mut cargo); + } + } + )+ } +} + +clean_crate_tree! { + Rustc, Mode::Rustc, "rustc-main"; + Std, Mode::Std, "sysroot"; +} + +fn clean(build: &Build, all: bool, stage: Option) { + if build.config.dry_run() { + return; + } + + rm_rf("tmp".as_ref()); + + // Clean the entire build directory + if all { + rm_rf(&build.out); + return; + } + + // Clean the target stage artifacts + if let Some(stage) = stage { + clean_specific_stage(build, stage); + return; + } + + // Follow the default behaviour + clean_default(build); +} + +fn clean_specific_stage(build: &Build, stage: u32) { + for host in &build.hosts { + let entries = match build.out.join(host.triple).read_dir() { + Ok(iter) => iter, + Err(_) => continue, + }; + + for entry in entries { + let entry = t!(entry); + let stage_prefix = format!("stage{}", stage); + + // if current entry is not related with the target stage, continue + if !entry.file_name().to_str().unwrap_or("").contains(&stage_prefix) { + continue; + } + + let path = t!(entry.path().canonicalize()); + rm_rf(&path); + } + } +} + +fn clean_default(build: &Build) { + rm_rf(&build.out.join("tmp")); + rm_rf(&build.out.join("dist")); + rm_rf(&build.out.join("rustfmt.stamp")); + + for host in &build.hosts { + let entries = match build.out.join(host.triple).read_dir() { + Ok(iter) => iter, + Err(_) => continue, + }; + + for entry in entries { + let entry = t!(entry); + if entry.file_name().to_str() == Some("llvm") { + continue; + } + let path = t!(entry.path().canonicalize()); + rm_rf(&path); + } + } +} + +fn rm_rf(path: &Path) { + match path.symlink_metadata() { + Err(e) => { + if e.kind() == ErrorKind::NotFound { + return; + } + panic!("failed to get metadata for file {}: {}", path.display(), e); + } + Ok(metadata) => { + if metadata.file_type().is_file() || metadata.file_type().is_symlink() { + do_op(path, "remove file", |p| { + fs::remove_file(p).or_else(|e| { + // Work around the fact that we cannot + // delete an executable while it runs on Windows. + #[cfg(windows)] + if e.kind() == std::io::ErrorKind::PermissionDenied + && p.file_name().and_then(std::ffi::OsStr::to_str) + == Some("bootstrap.exe") + { + eprintln!("WARNING: failed to delete '{}'.", p.display()); + return Ok(()); + } + Err(e) + }) + }); + return; + } + + for file in t!(fs::read_dir(path)) { + rm_rf(&t!(file).path()); + } + do_op(path, "remove dir", |p| { + fs::remove_dir(p).or_else(|e| { + // Check for dir not empty on Windows + // FIXME: Once `ErrorKind::DirectoryNotEmpty` is stabilized, + // match on `e.kind()` instead. + #[cfg(windows)] + if e.raw_os_error() == Some(145) { + return Ok(()); + } + + Err(e) + }) + }); + } + }; +} + +fn do_op(path: &Path, desc: &str, mut f: F) +where + F: FnMut(&Path) -> io::Result<()>, +{ + match f(path) { + Ok(()) => {} + // On windows we can't remove a readonly file, and git will often clone files as readonly. + // As a result, we have some special logic to remove readonly files on windows. + // This is also the reason that we can't use things like fs::remove_dir_all(). + Err(ref e) if cfg!(windows) && e.kind() == ErrorKind::PermissionDenied => { + let m = t!(path.symlink_metadata()); + let mut p = m.permissions(); + p.set_readonly(false); + t!(fs::set_permissions(path, p)); + f(path).unwrap_or_else(|e| { + // Delete symlinked directories on Windows + #[cfg(windows)] + if m.file_type().is_symlink() && path.is_dir() && fs::remove_dir(path).is_ok() { + return; + } + panic!("failed to {} {}: {}", desc, path.display(), e); + }); + } + Err(e) => { + panic!("failed to {} {}: {}", desc, path.display(), e); + } + } +} diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs new file mode 100644 index 000000000..7021a9543 --- /dev/null +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -0,0 +1,2069 @@ +//! Implementation of compiling various phases of the compiler and standard +//! library. +//! +//! This module contains some of the real meat in the rustbuild build system +//! which is where Cargo is used to compile the standard library, libtest, and +//! the compiler. This module is also responsible for assembling the sysroot as it +//! goes along from the output of the previous stage. + +use std::borrow::Cow; +use std::collections::HashSet; +use std::env; +use std::ffi::OsStr; +use std::fs; +use std::io::prelude::*; +use std::io::BufReader; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::str; + +use serde_derive::Deserialize; + +use crate::core::build_steps::dist; +use crate::core::build_steps::llvm; +use crate::core::build_steps::tool::SourceType; +use crate::core::builder::crate_description; +use crate::core::builder::Cargo; +use crate::core::builder::{Builder, Kind, PathSet, RunConfig, ShouldRun, Step, TaskPath}; +use crate::core::config::{DebuginfoLevel, LlvmLibunwind, RustcLto, TargetSelection}; +use crate::utils::cache::{Interned, INTERNER}; +use crate::utils::helpers::{ + exe, get_clang_cl_resource_dir, is_debug_info, is_dylib, output, symlink_dir, t, up_to_date, +}; +use crate::LLVM_TOOLS; +use crate::{CLang, Compiler, DependencyType, GitRepo, Mode}; +use filetime::FileTime; + +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Std { + pub target: TargetSelection, + pub compiler: Compiler, + /// Whether to build only a subset of crates in the standard library. + /// + /// This shouldn't be used from other steps; see the comment on [`Rustc`]. + crates: Interned>, + /// When using download-rustc, we need to use a new build of `std` for running unit tests of Std itself, + /// but we need to use the downloaded copy of std for linking to rustdoc. Allow this to be overriden by `builder.ensure` from other steps. + force_recompile: bool, + extra_rust_args: &'static [&'static str], +} + +impl Std { + pub fn new(compiler: Compiler, target: TargetSelection) -> Self { + Self { + target, + compiler, + crates: Default::default(), + force_recompile: false, + extra_rust_args: &[], + } + } + + pub fn force_recompile(compiler: Compiler, target: TargetSelection) -> Self { + Self { + target, + compiler, + crates: Default::default(), + force_recompile: true, + extra_rust_args: &[], + } + } + + pub fn new_with_extra_rust_args( + compiler: Compiler, + target: TargetSelection, + extra_rust_args: &'static [&'static str], + ) -> Self { + Self { + target, + compiler, + crates: Default::default(), + force_recompile: false, + extra_rust_args, + } + } +} + +impl Step for Std { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + // When downloading stage1, the standard library has already been copied to the sysroot, so + // there's no need to rebuild it. + let builder = run.builder; + run.crate_or_deps("sysroot") + .path("library") + .lazy_default_condition(Box::new(|| !builder.download_rustc())) + } + + fn make_run(run: RunConfig<'_>) { + // If the paths include "library", build the entire standard library. + let has_alias = + run.paths.iter().any(|set| set.assert_single_path().path.ends_with("library")); + let crates = if has_alias { Default::default() } else { run.cargo_crates_in_set() }; + + run.builder.ensure(Std { + compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), + target: run.target, + crates, + force_recompile: false, + extra_rust_args: &[], + }); + } + + /// Builds the standard library. + /// + /// This will build the standard library for a particular stage of the build + /// using the `compiler` targeting the `target` architecture. The artifacts + /// created will also be linked into the sysroot directory. + fn run(self, builder: &Builder<'_>) { + let target = self.target; + let compiler = self.compiler; + + // When using `download-rustc`, we already have artifacts for the host available. Don't + // recompile them. + if builder.download_rustc() && target == builder.build.build + // NOTE: the beta compiler may generate different artifacts than the downloaded compiler, so + // its artifacts can't be reused. + && compiler.stage != 0 + // This check is specific to testing std itself; see `test::Std` for more details. + && !self.force_recompile + { + cp_rustc_component_to_ci_sysroot( + builder, + compiler, + builder.config.ci_rust_std_contents(), + ); + return; + } + + if builder.config.keep_stage.contains(&compiler.stage) + || builder.config.keep_stage_std.contains(&compiler.stage) + { + builder.info("WARNING: Using a potentially old libstd. This may not behave well."); + + copy_third_party_objects(builder, &compiler, target); + copy_self_contained_objects(builder, &compiler, target); + + builder.ensure(StdLink::from_std(self, compiler)); + return; + } + + builder.update_submodule(&Path::new("library").join("stdarch")); + + // Profiler information requires LLVM's compiler-rt + if builder.config.profiler { + builder.update_submodule(&Path::new("src/llvm-project")); + } + + let mut target_deps = builder.ensure(StartupObjects { compiler, target }); + + let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); + if compiler_to_use != compiler { + builder.ensure(Std::new(compiler_to_use, target)); + let msg = if compiler_to_use.host == target { + format!( + "Uplifting library (stage{} -> stage{})", + compiler_to_use.stage, compiler.stage + ) + } else { + format!( + "Uplifting library (stage{}:{} -> stage{}:{})", + compiler_to_use.stage, compiler_to_use.host, compiler.stage, target + ) + }; + builder.info(&msg); + + // Even if we're not building std this stage, the new sysroot must + // still contain the third party objects needed by various targets. + copy_third_party_objects(builder, &compiler, target); + copy_self_contained_objects(builder, &compiler, target); + + builder.ensure(StdLink::from_std(self, compiler_to_use)); + return; + } + + target_deps.extend(copy_third_party_objects(builder, &compiler, target)); + target_deps.extend(copy_self_contained_objects(builder, &compiler, target)); + + // The LLD wrappers and `rust-lld` are self-contained linking components that can be + // necessary to link the stdlib on some targets. We'll also need to copy these binaries to + // the `stage0-sysroot` to ensure the linker is found when bootstrapping on such a target. + if compiler.stage == 0 && compiler.host == builder.config.build { + // We want to copy the host `bin` folder within the `rustlib` folder in the sysroot. + let src_sysroot_bin = builder + .rustc_snapshot_sysroot() + .join("lib") + .join("rustlib") + .join(compiler.host.triple) + .join("bin"); + if src_sysroot_bin.exists() { + let target_sysroot_bin = + builder.sysroot_libdir(compiler, target).parent().unwrap().join("bin"); + t!(fs::create_dir_all(&target_sysroot_bin)); + builder.cp_r(&src_sysroot_bin, &target_sysroot_bin); + } + } + + let mut cargo = builder.cargo(compiler, Mode::Std, SourceType::InTree, target, "build"); + std_cargo(builder, target, compiler.stage, &mut cargo); + for krate in &*self.crates { + cargo.arg("-p").arg(krate); + } + + // See src/bootstrap/synthetic_targets.rs + if target.is_synthetic() { + cargo.env("RUSTC_BOOTSTRAP_SYNTHETIC_TARGET", "1"); + } + for rustflag in self.extra_rust_args.into_iter() { + cargo.rustflag(rustflag); + } + + let _guard = builder.msg( + Kind::Build, + compiler.stage, + format_args!("library artifacts{}", crate_description(&self.crates)), + compiler.host, + target, + ); + run_cargo( + builder, + cargo, + vec![], + &libstd_stamp(builder, compiler, target), + target_deps, + false, + false, + ); + + builder.ensure(StdLink::from_std( + self, + builder.compiler(compiler.stage, builder.config.build), + )); + } +} + +fn copy_and_stamp( + builder: &Builder<'_>, + libdir: &Path, + sourcedir: &Path, + name: &str, + target_deps: &mut Vec<(PathBuf, DependencyType)>, + dependency_type: DependencyType, +) { + let target = libdir.join(name); + builder.copy(&sourcedir.join(name), &target); + + target_deps.push((target, dependency_type)); +} + +fn copy_llvm_libunwind(builder: &Builder<'_>, target: TargetSelection, libdir: &Path) -> PathBuf { + let libunwind_path = builder.ensure(llvm::Libunwind { target }); + let libunwind_source = libunwind_path.join("libunwind.a"); + let libunwind_target = libdir.join("libunwind.a"); + builder.copy(&libunwind_source, &libunwind_target); + libunwind_target +} + +/// Copies third party objects needed by various targets. +fn copy_third_party_objects( + builder: &Builder<'_>, + compiler: &Compiler, + target: TargetSelection, +) -> Vec<(PathBuf, DependencyType)> { + let mut target_deps = vec![]; + + if builder.config.sanitizers_enabled(target) && compiler.stage != 0 { + // The sanitizers are only copied in stage1 or above, + // to avoid creating dependency on LLVM. + target_deps.extend( + copy_sanitizers(builder, &compiler, target) + .into_iter() + .map(|d| (d, DependencyType::Target)), + ); + } + + if target == "x86_64-fortanix-unknown-sgx" + || builder.config.llvm_libunwind(target) == LlvmLibunwind::InTree + && (target.contains("linux") || target.contains("fuchsia")) + { + let libunwind_path = + copy_llvm_libunwind(builder, target, &builder.sysroot_libdir(*compiler, target)); + target_deps.push((libunwind_path, DependencyType::Target)); + } + + target_deps +} + +/// Copies third party objects needed by various targets for self-contained linkage. +fn copy_self_contained_objects( + builder: &Builder<'_>, + compiler: &Compiler, + target: TargetSelection, +) -> Vec<(PathBuf, DependencyType)> { + let libdir_self_contained = builder.sysroot_libdir(*compiler, target).join("self-contained"); + t!(fs::create_dir_all(&libdir_self_contained)); + let mut target_deps = vec![]; + + // Copies the libc and CRT objects. + // + // rustc historically provides a more self-contained installation for musl targets + // not requiring the presence of a native musl toolchain. For example, it can fall back + // to using gcc from a glibc-targeting toolchain for linking. + // To do that we have to distribute musl startup objects as a part of Rust toolchain + // and link with them manually in the self-contained mode. + if target.contains("musl") && !target.contains("unikraft") { + let srcdir = builder.musl_libdir(target).unwrap_or_else(|| { + panic!("Target {:?} does not have a \"musl-libdir\" key", target.triple) + }); + for &obj in &["libc.a", "crt1.o", "Scrt1.o", "rcrt1.o", "crti.o", "crtn.o"] { + copy_and_stamp( + builder, + &libdir_self_contained, + &srcdir, + obj, + &mut target_deps, + DependencyType::TargetSelfContained, + ); + } + let crt_path = builder.ensure(llvm::CrtBeginEnd { target }); + for &obj in &["crtbegin.o", "crtbeginS.o", "crtend.o", "crtendS.o"] { + let src = crt_path.join(obj); + let target = libdir_self_contained.join(obj); + builder.copy(&src, &target); + target_deps.push((target, DependencyType::TargetSelfContained)); + } + + if !target.starts_with("s390x") { + let libunwind_path = copy_llvm_libunwind(builder, target, &libdir_self_contained); + target_deps.push((libunwind_path, DependencyType::TargetSelfContained)); + } + } else if target.contains("-wasi") { + let srcdir = builder + .wasi_root(target) + .unwrap_or_else(|| { + panic!("Target {:?} does not have a \"wasi-root\" key", target.triple) + }) + .join("lib") + .join(target.to_string().replace("-preview1", "")); + for &obj in &["libc.a", "crt1-command.o", "crt1-reactor.o"] { + copy_and_stamp( + builder, + &libdir_self_contained, + &srcdir, + obj, + &mut target_deps, + DependencyType::TargetSelfContained, + ); + } + } else if target.ends_with("windows-gnu") { + for obj in ["crt2.o", "dllcrt2.o"].iter() { + let src = compiler_file(builder, &builder.cc(target), target, CLang::C, obj); + let target = libdir_self_contained.join(obj); + builder.copy(&src, &target); + target_deps.push((target, DependencyType::TargetSelfContained)); + } + } + + target_deps +} + +/// Configure cargo to compile the standard library, adding appropriate env vars +/// and such. +pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, cargo: &mut Cargo) { + if let Some(target) = env::var_os("MACOSX_STD_DEPLOYMENT_TARGET") { + cargo.env("MACOSX_DEPLOYMENT_TARGET", target); + } + + if let Some(path) = builder.config.profiler_path(target) { + cargo.env("LLVM_PROFILER_RT_LIB", path); + } + + // Determine if we're going to compile in optimized C intrinsics to + // the `compiler-builtins` crate. These intrinsics live in LLVM's + // `compiler-rt` repository, but our `src/llvm-project` submodule isn't + // always checked out, so we need to conditionally look for this. (e.g. if + // an external LLVM is used we skip the LLVM submodule checkout). + // + // Note that this shouldn't affect the correctness of `compiler-builtins`, + // but only its speed. Some intrinsics in C haven't been translated to Rust + // yet but that's pretty rare. Other intrinsics have optimized + // implementations in C which have only had slower versions ported to Rust, + // so we favor the C version where we can, but it's not critical. + // + // If `compiler-rt` is available ensure that the `c` feature of the + // `compiler-builtins` crate is enabled and it's configured to learn where + // `compiler-rt` is located. + let compiler_builtins_root = builder.src.join("src/llvm-project/compiler-rt"); + let compiler_builtins_c_feature = if compiler_builtins_root.exists() { + // Note that `libprofiler_builtins/build.rs` also computes this so if + // you're changing something here please also change that. + cargo.env("RUST_COMPILER_RT_ROOT", &compiler_builtins_root); + " compiler-builtins-c" + } else { + "" + }; + + // `libtest` uses this to know whether or not to support + // `-Zunstable-options`. + if !builder.unstable_features() { + cargo.env("CFG_DISABLE_UNSTABLE_FEATURES", "1"); + } + + let mut features = String::new(); + + if builder.no_std(target) == Some(true) { + features += " compiler-builtins-mem"; + if !target.starts_with("bpf") { + features.push_str(compiler_builtins_c_feature); + } + + // for no-std targets we only compile a few no_std crates + cargo + .args(&["-p", "alloc"]) + .arg("--manifest-path") + .arg(builder.src.join("library/alloc/Cargo.toml")) + .arg("--features") + .arg(features); + } else { + features += &builder.std_features(target); + features.push_str(compiler_builtins_c_feature); + + cargo + .arg("--features") + .arg(features) + .arg("--manifest-path") + .arg(builder.src.join("library/sysroot/Cargo.toml")); + + // Help the libc crate compile by assisting it in finding various + // sysroot native libraries. + if target.contains("musl") { + if let Some(p) = builder.musl_libdir(target) { + let root = format!("native={}", p.to_str().unwrap()); + cargo.rustflag("-L").rustflag(&root); + } + } + + if target.contains("-wasi") { + if let Some(p) = builder.wasi_root(target) { + let root = format!( + "native={}/lib/{}", + p.to_str().unwrap(), + target.to_string().replace("-preview1", "") + ); + cargo.rustflag("-L").rustflag(&root); + } + } + } + + // By default, rustc uses `-Cembed-bitcode=yes`, and Cargo overrides that + // with `-Cembed-bitcode=no` for non-LTO builds. However, libstd must be + // built with bitcode so that the produced rlibs can be used for both LTO + // builds (which use bitcode) and non-LTO builds (which use object code). + // So we override the override here! + // + // But we don't bother for the stage 0 compiler because it's never used + // with LTO. + if stage >= 1 { + cargo.rustflag("-Cembed-bitcode=yes"); + } + if builder.config.rust_lto == RustcLto::Off { + cargo.rustflag("-Clto=off"); + } + + // By default, rustc does not include unwind tables unless they are required + // for a particular target. They are not required by RISC-V targets, but + // compiling the standard library with them means that users can get + // backtraces without having to recompile the standard library themselves. + // + // This choice was discussed in https://github.com/rust-lang/rust/pull/69890 + if target.contains("riscv") { + cargo.rustflag("-Cforce-unwind-tables=yes"); + } + + let html_root = + format!("-Zcrate-attr=doc(html_root_url=\"{}/\")", builder.doc_rust_lang_org_channel(),); + cargo.rustflag(&html_root); + cargo.rustdocflag(&html_root); + + cargo.rustdocflag("-Zcrate-attr=warn(rust_2018_idioms)"); +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +struct StdLink { + pub compiler: Compiler, + pub target_compiler: Compiler, + pub target: TargetSelection, + /// Not actually used; only present to make sure the cache invalidation is correct. + crates: Interned>, + /// See [`Std::force_recompile`]. + force_recompile: bool, +} + +impl StdLink { + fn from_std(std: Std, host_compiler: Compiler) -> Self { + Self { + compiler: host_compiler, + target_compiler: std.compiler, + target: std.target, + crates: std.crates, + force_recompile: std.force_recompile, + } + } +} + +impl Step for StdLink { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Link all libstd rlibs/dylibs into the sysroot location. + /// + /// Links those artifacts generated by `compiler` to the `stage` compiler's + /// sysroot for the specified `host` and `target`. + /// + /// Note that this assumes that `compiler` has already generated the libstd + /// libraries for `target`, and this method will find them in the relevant + /// output directory. + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target_compiler = self.target_compiler; + let target = self.target; + + // NOTE: intentionally does *not* check `target == builder.build` to avoid having to add the same check in `test::Crate`. + let (libdir, hostdir) = if self.force_recompile && builder.download_rustc() { + // NOTE: copies part of `sysroot_libdir` to avoid having to add a new `force_recompile` argument there too + let lib = builder.sysroot_libdir_relative(self.compiler); + let sysroot = builder.ensure(crate::core::build_steps::compile::Sysroot { + compiler: self.compiler, + force_recompile: self.force_recompile, + }); + let libdir = sysroot.join(lib).join("rustlib").join(target.triple).join("lib"); + let hostdir = sysroot.join(lib).join("rustlib").join(compiler.host.triple).join("lib"); + (INTERNER.intern_path(libdir), INTERNER.intern_path(hostdir)) + } else { + let libdir = builder.sysroot_libdir(target_compiler, target); + let hostdir = builder.sysroot_libdir(target_compiler, compiler.host); + (libdir, hostdir) + }; + + add_to_sysroot(builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target)); + + // Special case for stage0, to make `rustup toolchain link` and `x dist --stage 0` + // work for stage0-sysroot. We only do this if the stage0 compiler comes from beta, + // and is not set to a custom path. + if compiler.stage == 0 + && builder + .build + .config + .initial_rustc + .starts_with(builder.out.join(&compiler.host.triple).join("stage0/bin")) + { + // Copy bin files from stage0/bin to stage0-sysroot/bin + let sysroot = builder.out.join(&compiler.host.triple).join("stage0-sysroot"); + + let host = compiler.host.triple; + let stage0_bin_dir = builder.out.join(&host).join("stage0/bin"); + let sysroot_bin_dir = sysroot.join("bin"); + t!(fs::create_dir_all(&sysroot_bin_dir)); + builder.cp_r(&stage0_bin_dir, &sysroot_bin_dir); + + // Copy all *.so files from stage0/lib to stage0-sysroot/lib + let stage0_lib_dir = builder.out.join(&host).join("stage0/lib"); + if let Ok(files) = fs::read_dir(&stage0_lib_dir) { + for file in files { + let file = t!(file); + let path = file.path(); + if path.is_file() && is_dylib(&file.file_name().into_string().unwrap()) { + builder.copy(&path, &sysroot.join("lib").join(path.file_name().unwrap())); + } + } + } + + // Copy codegen-backends from stage0 + let sysroot_codegen_backends = builder.sysroot_codegen_backends(compiler); + t!(fs::create_dir_all(&sysroot_codegen_backends)); + let stage0_codegen_backends = builder + .out + .join(&host) + .join("stage0/lib/rustlib") + .join(&host) + .join("codegen-backends"); + builder.cp_r(&stage0_codegen_backends, &sysroot_codegen_backends); + } + } +} + +/// Copies sanitizer runtime libraries into target libdir. +fn copy_sanitizers( + builder: &Builder<'_>, + compiler: &Compiler, + target: TargetSelection, +) -> Vec { + let runtimes: Vec = builder.ensure(llvm::Sanitizers { target }); + + if builder.config.dry_run() { + return Vec::new(); + } + + let mut target_deps = Vec::new(); + let libdir = builder.sysroot_libdir(*compiler, target); + + for runtime in &runtimes { + let dst = libdir.join(&runtime.name); + builder.copy(&runtime.path, &dst); + + // The `aarch64-apple-ios-macabi` and `x86_64-apple-ios-macabi` are also supported for + // sanitizers, but they share a sanitizer runtime with `${arch}-apple-darwin`, so we do + // not list them here to rename and sign the runtime library. + if target == "x86_64-apple-darwin" + || target == "aarch64-apple-darwin" + || target == "aarch64-apple-ios" + || target == "aarch64-apple-ios-sim" + || target == "x86_64-apple-ios" + { + // Update the library’s install name to reflect that it has been renamed. + apple_darwin_update_library_name(&dst, &format!("@rpath/{}", &runtime.name)); + // Upon renaming the install name, the code signature of the file will invalidate, + // so we will sign it again. + apple_darwin_sign_file(&dst); + } + + target_deps.push(dst); + } + + target_deps +} + +fn apple_darwin_update_library_name(library_path: &Path, new_name: &str) { + let status = Command::new("install_name_tool") + .arg("-id") + .arg(new_name) + .arg(library_path) + .status() + .expect("failed to execute `install_name_tool`"); + assert!(status.success()); +} + +fn apple_darwin_sign_file(file_path: &Path) { + let status = Command::new("codesign") + .arg("-f") // Force to rewrite the existing signature + .arg("-s") + .arg("-") + .arg(file_path) + .status() + .expect("failed to execute `codesign`"); + assert!(status.success()); +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct StartupObjects { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for StartupObjects { + type Output = Vec<(PathBuf, DependencyType)>; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("library/rtstartup") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(StartupObjects { + compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), + target: run.target, + }); + } + + /// Builds and prepare startup objects like rsbegin.o and rsend.o + /// + /// These are primarily used on Windows right now for linking executables/dlls. + /// They don't require any library support as they're just plain old object + /// files, so we just use the nightly snapshot compiler to always build them (as + /// no other compilers are guaranteed to be available). + fn run(self, builder: &Builder<'_>) -> Vec<(PathBuf, DependencyType)> { + let for_compiler = self.compiler; + let target = self.target; + if !target.ends_with("windows-gnu") { + return vec![]; + } + + let mut target_deps = vec![]; + + let src_dir = &builder.src.join("library").join("rtstartup"); + let dst_dir = &builder.native_dir(target).join("rtstartup"); + let sysroot_dir = &builder.sysroot_libdir(for_compiler, target); + t!(fs::create_dir_all(dst_dir)); + + for file in &["rsbegin", "rsend"] { + let src_file = &src_dir.join(file.to_string() + ".rs"); + let dst_file = &dst_dir.join(file.to_string() + ".o"); + if !up_to_date(src_file, dst_file) { + let mut cmd = Command::new(&builder.initial_rustc); + cmd.env("RUSTC_BOOTSTRAP", "1"); + if !builder.local_rebuild { + // a local_rebuild compiler already has stage1 features + cmd.arg("--cfg").arg("bootstrap"); + } + builder.run( + cmd.arg("--target") + .arg(target.rustc_target_arg()) + .arg("--emit=obj") + .arg("-o") + .arg(dst_file) + .arg(src_file), + ); + } + + let target = sysroot_dir.join((*file).to_string() + ".o"); + builder.copy(dst_file, &target); + target_deps.push((target, DependencyType::Target)); + } + + target_deps + } +} + +fn cp_rustc_component_to_ci_sysroot( + builder: &Builder<'_>, + compiler: Compiler, + contents: Vec, +) { + let sysroot = builder.ensure(Sysroot { compiler, force_recompile: false }); + let ci_rustc_dir = builder.config.ci_rustc_dir(); + + for file in contents { + let src = ci_rustc_dir.join(&file); + let dst = sysroot.join(file); + if src.is_dir() { + t!(fs::create_dir_all(dst)); + } else { + builder.copy(&src, &dst); + } + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Rustc { + pub target: TargetSelection, + pub compiler: Compiler, + /// Whether to build a subset of crates, rather than the whole compiler. + /// + /// This should only be requested by the user, not used within rustbuild itself. + /// Using it within rustbuild can lead to confusing situation where lints are replayed + /// in two different steps. + crates: Interned>, +} + +impl Rustc { + pub fn new(compiler: Compiler, target: TargetSelection) -> Self { + Self { target, compiler, crates: Default::default() } + } +} + +impl Step for Rustc { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let mut crates = run.builder.in_tree_crates("rustc-main", None); + for (i, krate) in crates.iter().enumerate() { + // We can't allow `build rustc` as an alias for this Step, because that's reserved by `Assemble`. + // Ideally Assemble would use `build compiler` instead, but that seems too confusing to be worth the breaking change. + if krate.name == "rustc-main" { + crates.swap_remove(i); + break; + } + } + run.crates(crates) + } + + fn make_run(run: RunConfig<'_>) { + let crates = run.cargo_crates_in_set(); + run.builder.ensure(Rustc { + compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), + target: run.target, + crates, + }); + } + + /// Builds the compiler. + /// + /// This will build the compiler for a particular stage of the build using + /// the `compiler` targeting the `target` architecture. The artifacts + /// created will also be linked into the sysroot directory. + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target = self.target; + + // NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler, + // so its artifacts can't be reused. + if builder.download_rustc() && compiler.stage != 0 { + // Copy the existing artifacts instead of rebuilding them. + // NOTE: this path is only taken for tools linking to rustc-dev (including ui-fulldeps tests). + cp_rustc_component_to_ci_sysroot( + builder, + compiler, + builder.config.ci_rustc_dev_contents(), + ); + return; + } + + builder.ensure(Std::new(compiler, target)); + + if builder.config.keep_stage.contains(&compiler.stage) { + builder.info("WARNING: Using a potentially old librustc. This may not behave well."); + builder.info("WARNING: Use `--keep-stage-std` if you want to rebuild the compiler when it changes"); + builder.ensure(RustcLink::from_rustc(self, compiler)); + return; + } + + let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); + if compiler_to_use != compiler { + builder.ensure(Rustc::new(compiler_to_use, target)); + let msg = if compiler_to_use.host == target { + format!( + "Uplifting rustc (stage{} -> stage{})", + compiler_to_use.stage, + compiler.stage + 1 + ) + } else { + format!( + "Uplifting rustc (stage{}:{} -> stage{}:{})", + compiler_to_use.stage, + compiler_to_use.host, + compiler.stage + 1, + target + ) + }; + builder.info(&msg); + builder.ensure(RustcLink::from_rustc(self, compiler_to_use)); + return; + } + + // Ensure that build scripts and proc macros have a std / libproc_macro to link against. + builder.ensure(Std::new( + builder.compiler(self.compiler.stage, builder.config.build), + builder.config.build, + )); + + let mut cargo = builder.cargo(compiler, Mode::Rustc, SourceType::InTree, target, "build"); + rustc_cargo(builder, &mut cargo, target, compiler.stage); + + if builder.config.rust_profile_use.is_some() + && builder.config.rust_profile_generate.is_some() + { + panic!("Cannot use and generate PGO profiles at the same time"); + } + + // With LLD, we can use ICF (identical code folding) to reduce the executable size + // of librustc_driver/rustc and to improve i-cache utilization. + // + // -Wl,[link options] doesn't work on MSVC. However, /OPT:ICF (technically /OPT:REF,ICF) + // is already on by default in MSVC optimized builds, which is interpreted as --icf=all: + // https://github.com/llvm/llvm-project/blob/3329cec2f79185bafd678f310fafadba2a8c76d2/lld/COFF/Driver.cpp#L1746 + // https://github.com/rust-lang/rust/blob/f22819bcce4abaff7d1246a56eec493418f9f4ee/compiler/rustc_codegen_ssa/src/back/linker.rs#L827 + if builder.config.use_lld && !compiler.host.contains("msvc") { + cargo.rustflag("-Clink-args=-Wl,--icf=all"); + } + + let is_collecting = if let Some(path) = &builder.config.rust_profile_generate { + if compiler.stage == 1 { + cargo.rustflag(&format!("-Cprofile-generate={path}")); + // Apparently necessary to avoid overflowing the counters during + // a Cargo build profile + cargo.rustflag("-Cllvm-args=-vp-counters-per-site=4"); + true + } else { + false + } + } else if let Some(path) = &builder.config.rust_profile_use { + if compiler.stage == 1 { + cargo.rustflag(&format!("-Cprofile-use={path}")); + cargo.rustflag("-Cllvm-args=-pgo-warn-missing-function"); + true + } else { + false + } + } else { + false + }; + if is_collecting { + // Ensure paths to Rust sources are relative, not absolute. + cargo.rustflag(&format!( + "-Cllvm-args=-static-func-strip-dirname-prefix={}", + builder.config.src.components().count() + )); + } + + // We currently don't support cross-crate LTO in stage0. This also isn't hugely necessary + // and may just be a time sink. + if compiler.stage != 0 { + match builder.config.rust_lto { + RustcLto::Thin | RustcLto::Fat => { + // Since using LTO for optimizing dylibs is currently experimental, + // we need to pass -Zdylib-lto. + cargo.rustflag("-Zdylib-lto"); + // Cargo by default passes `-Cembed-bitcode=no` and doesn't pass `-Clto` when + // compiling dylibs (and their dependencies), even when LTO is enabled for the + // crate. Therefore, we need to override `-Clto` and `-Cembed-bitcode` here. + let lto_type = match builder.config.rust_lto { + RustcLto::Thin => "thin", + RustcLto::Fat => "fat", + _ => unreachable!(), + }; + cargo.rustflag(&format!("-Clto={lto_type}")); + cargo.rustflag("-Cembed-bitcode=yes"); + } + RustcLto::ThinLocal => { /* Do nothing, this is the default */ } + RustcLto::Off => { + cargo.rustflag("-Clto=off"); + } + } + } else if builder.config.rust_lto == RustcLto::Off { + cargo.rustflag("-Clto=off"); + } + + for krate in &*self.crates { + cargo.arg("-p").arg(krate); + } + + if builder.build.config.enable_bolt_settings && compiler.stage == 1 { + // Relocations are required for BOLT to work. + cargo.env("RUSTC_BOLT_LINK_FLAGS", "1"); + } + + let _guard = builder.msg_sysroot_tool( + Kind::Build, + compiler.stage, + format_args!("compiler artifacts{}", crate_description(&self.crates)), + compiler.host, + target, + ); + let stamp = librustc_stamp(builder, compiler, target); + run_cargo( + builder, + cargo, + vec![], + &stamp, + vec![], + false, + true, // Only ship rustc_driver.so and .rmeta files, not all intermediate .rlib files. + ); + + // When building `librustc_driver.so` (like `libLLVM.so`) on linux, it can contain + // unexpected debuginfo from dependencies, for example from the C++ standard library used in + // our LLVM wrapper. Unless we're explicitly requesting `librustc_driver` to be built with + // debuginfo (via the debuginfo level of the executables using it): strip this debuginfo + // away after the fact. + if builder.config.rust_debuginfo_level_rustc == DebuginfoLevel::None + && builder.config.rust_debuginfo_level_tools == DebuginfoLevel::None + { + let target_root_dir = stamp.parent().unwrap(); + let rustc_driver = target_root_dir.join("librustc_driver.so"); + strip_debug(builder, target, &rustc_driver); + } + + builder.ensure(RustcLink::from_rustc( + self, + builder.compiler(compiler.stage, builder.config.build), + )); + } +} + +pub fn rustc_cargo(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelection, stage: u32) { + cargo + .arg("--features") + .arg(builder.rustc_features(builder.kind)) + .arg("--manifest-path") + .arg(builder.src.join("compiler/rustc/Cargo.toml")); + + cargo.rustdocflag("-Zcrate-attr=warn(rust_2018_idioms)"); + + rustc_cargo_env(builder, cargo, target, stage); +} + +pub fn rustc_cargo_env( + builder: &Builder<'_>, + cargo: &mut Cargo, + target: TargetSelection, + stage: u32, +) { + // Set some configuration variables picked up by build scripts and + // the compiler alike + cargo + .env("CFG_RELEASE", builder.rust_release()) + .env("CFG_RELEASE_CHANNEL", &builder.config.channel) + .env("CFG_VERSION", builder.rust_version()); + + if let Some(backend) = builder.config.default_codegen_backend() { + cargo.env("CFG_DEFAULT_CODEGEN_BACKEND", backend); + } + + let libdir_relative = builder.config.libdir_relative().unwrap_or_else(|| Path::new("lib")); + let target_config = builder.config.target_config.get(&target); + + cargo.env("CFG_LIBDIR_RELATIVE", libdir_relative); + + if let Some(ref ver_date) = builder.rust_info().commit_date() { + cargo.env("CFG_VER_DATE", ver_date); + } + if let Some(ref ver_hash) = builder.rust_info().sha() { + cargo.env("CFG_VER_HASH", ver_hash); + } + if !builder.unstable_features() { + cargo.env("CFG_DISABLE_UNSTABLE_FEATURES", "1"); + } + + // Prefer the current target's own default_linker, else a globally + // specified one. + if let Some(s) = target_config.and_then(|c| c.default_linker.as_ref()) { + cargo.env("CFG_DEFAULT_LINKER", s); + } else if let Some(ref s) = builder.config.rustc_default_linker { + cargo.env("CFG_DEFAULT_LINKER", s); + } + + if builder.config.rustc_parallel { + // keep in sync with `bootstrap/lib.rs:Build::rustc_features` + // `cfg` option for rustc, `features` option for cargo, for conditional compilation + cargo.rustflag("--cfg=parallel_compiler"); + cargo.rustdocflag("--cfg=parallel_compiler"); + } + if builder.config.rust_verify_llvm_ir { + cargo.env("RUSTC_VERIFY_LLVM_IR", "1"); + } + + // Note that this is disabled if LLVM itself is disabled or we're in a check + // build. If we are in a check build we still go ahead here presuming we've + // detected that LLVM is already built and good to go which helps prevent + // busting caches (e.g. like #71152). + if builder.config.llvm_enabled() { + let building_is_expensive = + crate::core::build_steps::llvm::prebuilt_llvm_config(builder, target).is_err(); + // `top_stage == stage` might be false for `check --stage 1`, if we are building the stage 1 compiler + let can_skip_build = builder.kind == Kind::Check && builder.top_stage == stage; + let should_skip_build = building_is_expensive && can_skip_build; + if !should_skip_build { + rustc_llvm_env(builder, cargo, target) + } + } +} + +/// Pass down configuration from the LLVM build into the build of +/// rustc_llvm and rustc_codegen_llvm. +fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelection) { + let target_config = builder.config.target_config.get(&target); + + if builder.is_rust_llvm(target) { + cargo.env("LLVM_RUSTLLVM", "1"); + } + let llvm::LlvmResult { llvm_config, .. } = builder.ensure(llvm::Llvm { target }); + cargo.env("LLVM_CONFIG", &llvm_config); + if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) { + cargo.env("CFG_LLVM_ROOT", s); + } + + // Some LLVM linker flags (-L and -l) may be needed to link `rustc_llvm`. Its build script + // expects these to be passed via the `LLVM_LINKER_FLAGS` env variable, separated by + // whitespace. + // + // For example: + // - on windows, when `clang-cl` is used with instrumentation, we need to manually add + // clang's runtime library resource directory so that the profiler runtime library can be + // found. This is to avoid the linker errors about undefined references to + // `__llvm_profile_instrument_memop` when linking `rustc_driver`. + let mut llvm_linker_flags = String::new(); + if builder.config.llvm_profile_generate && target.contains("msvc") { + if let Some(ref clang_cl_path) = builder.config.llvm_clang_cl { + // Add clang's runtime library directory to the search path + let clang_rt_dir = get_clang_cl_resource_dir(clang_cl_path); + llvm_linker_flags.push_str(&format!("-L{}", clang_rt_dir.display())); + } + } + + // The config can also specify its own llvm linker flags. + if let Some(ref s) = builder.config.llvm_ldflags { + if !llvm_linker_flags.is_empty() { + llvm_linker_flags.push_str(" "); + } + llvm_linker_flags.push_str(s); + } + + // Set the linker flags via the env var that `rustc_llvm`'s build script will read. + if !llvm_linker_flags.is_empty() { + cargo.env("LLVM_LINKER_FLAGS", llvm_linker_flags); + } + + // Building with a static libstdc++ is only supported on linux right now, + // not for MSVC or macOS + if builder.config.llvm_static_stdcpp + && !target.contains("freebsd") + && !target.contains("msvc") + && !target.contains("apple") + && !target.contains("solaris") + { + let file = compiler_file( + builder, + &builder.cxx(target).unwrap(), + target, + CLang::Cxx, + "libstdc++.a", + ); + cargo.env("LLVM_STATIC_STDCPP", file); + } + if builder.llvm_link_shared() { + cargo.env("LLVM_LINK_SHARED", "1"); + } + if builder.config.llvm_use_libcxx { + cargo.env("LLVM_USE_LIBCXX", "1"); + } + if builder.config.llvm_optimize && !builder.config.llvm_release_debuginfo { + cargo.env("LLVM_NDEBUG", "1"); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +struct RustcLink { + pub compiler: Compiler, + pub target_compiler: Compiler, + pub target: TargetSelection, + /// Not actually used; only present to make sure the cache invalidation is correct. + crates: Interned>, +} + +impl RustcLink { + fn from_rustc(rustc: Rustc, host_compiler: Compiler) -> Self { + Self { + compiler: host_compiler, + target_compiler: rustc.compiler, + target: rustc.target, + crates: rustc.crates, + } + } +} + +impl Step for RustcLink { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Same as `std_link`, only for librustc + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target_compiler = self.target_compiler; + let target = self.target; + add_to_sysroot( + builder, + &builder.sysroot_libdir(target_compiler, target), + &builder.sysroot_libdir(target_compiler, compiler.host), + &librustc_stamp(builder, compiler, target), + ); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct CodegenBackend { + pub target: TargetSelection, + pub compiler: Compiler, + pub backend: Interned, +} + +fn needs_codegen_config(run: &RunConfig<'_>) -> bool { + let mut needs_codegen_cfg = false; + for path_set in &run.paths { + needs_codegen_cfg = match path_set { + PathSet::Set(set) => set.iter().any(|p| is_codegen_cfg_needed(p, run)), + PathSet::Suite(suite) => is_codegen_cfg_needed(&suite, run), + } + } + needs_codegen_cfg +} + +pub(crate) const CODEGEN_BACKEND_PREFIX: &str = "rustc_codegen_"; + +fn is_codegen_cfg_needed(path: &TaskPath, run: &RunConfig<'_>) -> bool { + if path.path.to_str().unwrap().contains(&CODEGEN_BACKEND_PREFIX) { + let mut needs_codegen_backend_config = true; + for &backend in &run.builder.config.rust_codegen_backends { + if path + .path + .to_str() + .unwrap() + .ends_with(&(CODEGEN_BACKEND_PREFIX.to_owned() + &backend)) + { + needs_codegen_backend_config = false; + } + } + if needs_codegen_backend_config { + run.builder.info( + "WARNING: no codegen-backends config matched the requested path to build a codegen backend. \ + HELP: add backend to codegen-backends in config.toml.", + ); + return true; + } + } + + return false; +} + +impl Step for CodegenBackend { + type Output = (); + const ONLY_HOSTS: bool = true; + // Only the backends specified in the `codegen-backends` entry of `config.toml` are built. + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.paths(&["compiler/rustc_codegen_cranelift", "compiler/rustc_codegen_gcc"]) + } + + fn make_run(run: RunConfig<'_>) { + if needs_codegen_config(&run) { + return; + } + + for &backend in &run.builder.config.rust_codegen_backends { + if backend == "llvm" { + continue; // Already built as part of rustc + } + + run.builder.ensure(CodegenBackend { + target: run.target, + compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), + backend, + }); + } + } + + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target = self.target; + let backend = self.backend; + + builder.ensure(Rustc::new(compiler, target)); + + if builder.config.keep_stage.contains(&compiler.stage) { + builder.info( + "WARNING: Using a potentially old codegen backend. \ + This may not behave well.", + ); + // Codegen backends are linked separately from this step today, so we don't do + // anything here. + return; + } + + let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); + if compiler_to_use != compiler { + builder.ensure(CodegenBackend { compiler: compiler_to_use, target, backend }); + return; + } + + let out_dir = builder.cargo_out(compiler, Mode::Codegen, target); + + let mut cargo = builder.cargo(compiler, Mode::Codegen, SourceType::InTree, target, "build"); + cargo + .arg("--manifest-path") + .arg(builder.src.join(format!("compiler/rustc_codegen_{backend}/Cargo.toml"))); + rustc_cargo_env(builder, &mut cargo, target, compiler.stage); + + let tmp_stamp = out_dir.join(".tmp.stamp"); + + let _guard = builder.msg_build(compiler, format_args!("codegen backend {backend}"), target); + let files = run_cargo(builder, cargo, vec![], &tmp_stamp, vec![], false, false); + if builder.config.dry_run() { + return; + } + let mut files = files.into_iter().filter(|f| { + let filename = f.file_name().unwrap().to_str().unwrap(); + is_dylib(filename) && filename.contains("rustc_codegen_") + }); + let codegen_backend = match files.next() { + Some(f) => f, + None => panic!("no dylibs built for codegen backend?"), + }; + if let Some(f) = files.next() { + panic!( + "codegen backend built two dylibs:\n{}\n{}", + codegen_backend.display(), + f.display() + ); + } + let stamp = codegen_backend_stamp(builder, compiler, target, backend); + let codegen_backend = codegen_backend.to_str().unwrap(); + t!(fs::write(&stamp, &codegen_backend)); + } +} + +/// Creates the `codegen-backends` folder for a compiler that's about to be +/// assembled as a complete compiler. +/// +/// This will take the codegen artifacts produced by `compiler` and link them +/// into an appropriate location for `target_compiler` to be a functional +/// compiler. +fn copy_codegen_backends_to_sysroot( + builder: &Builder<'_>, + compiler: Compiler, + target_compiler: Compiler, +) { + let target = target_compiler.host; + + // Note that this step is different than all the other `*Link` steps in + // that it's not assembling a bunch of libraries but rather is primarily + // moving the codegen backend into place. The codegen backend of rustc is + // not linked into the main compiler by default but is rather dynamically + // selected at runtime for inclusion. + // + // Here we're looking for the output dylib of the `CodegenBackend` step and + // we're copying that into the `codegen-backends` folder. + let dst = builder.sysroot_codegen_backends(target_compiler); + t!(fs::create_dir_all(&dst), dst); + + if builder.config.dry_run() { + return; + } + + for backend in builder.config.rust_codegen_backends.iter() { + if backend == "llvm" { + continue; // Already built as part of rustc + } + + let stamp = codegen_backend_stamp(builder, compiler, target, *backend); + let dylib = t!(fs::read_to_string(&stamp)); + let file = Path::new(&dylib); + let filename = file.file_name().unwrap().to_str().unwrap(); + // change `librustc_codegen_cranelift-xxxxxx.so` to + // `librustc_codegen_cranelift-release.so` + let target_filename = { + let dash = filename.find('-').unwrap(); + let dot = filename.find('.').unwrap(); + format!("{}-{}{}", &filename[..dash], builder.rust_release(), &filename[dot..]) + }; + builder.copy(&file, &dst.join(target_filename)); + } +} + +/// Cargo's output path for the standard library in a given stage, compiled +/// by a particular compiler for the specified target. +pub fn libstd_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { + builder.cargo_out(compiler, Mode::Std, target).join(".libstd.stamp") +} + +/// Cargo's output path for librustc in a given stage, compiled by a particular +/// compiler for the specified target. +pub fn librustc_stamp( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, +) -> PathBuf { + builder.cargo_out(compiler, Mode::Rustc, target).join(".librustc.stamp") +} + +/// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular +/// compiler for the specified target and backend. +fn codegen_backend_stamp( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, + backend: Interned, +) -> PathBuf { + builder + .cargo_out(compiler, Mode::Codegen, target) + .join(format!(".librustc_codegen_{backend}.stamp")) +} + +pub fn compiler_file( + builder: &Builder<'_>, + compiler: &Path, + target: TargetSelection, + c: CLang, + file: &str, +) -> PathBuf { + if builder.config.dry_run() { + return PathBuf::new(); + } + let mut cmd = Command::new(compiler); + cmd.args(builder.cflags(target, GitRepo::Rustc, c)); + cmd.arg(format!("-print-file-name={file}")); + let out = output(&mut cmd); + PathBuf::from(out.trim()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Sysroot { + pub compiler: Compiler, + /// See [`Std::force_recompile`]. + force_recompile: bool, +} + +impl Sysroot { + pub(crate) fn new(compiler: Compiler) -> Self { + Sysroot { compiler, force_recompile: false } + } +} + +impl Step for Sysroot { + type Output = Interned; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Returns the sysroot for the `compiler` specified that *this build system + /// generates*. + /// + /// That is, the sysroot for the stage0 compiler is not what the compiler + /// thinks it is by default, but it's the same as the default for stages + /// 1-3. + fn run(self, builder: &Builder<'_>) -> Interned { + let compiler = self.compiler; + let host_dir = builder.out.join(&compiler.host.triple); + + let sysroot_dir = |stage| { + if stage == 0 { + host_dir.join("stage0-sysroot") + } else if self.force_recompile && stage == compiler.stage { + host_dir.join(format!("stage{stage}-test-sysroot")) + } else if builder.download_rustc() && compiler.stage != builder.top_stage { + host_dir.join("ci-rustc-sysroot") + } else { + host_dir.join(format!("stage{}", stage)) + } + }; + let sysroot = sysroot_dir(compiler.stage); + + builder.verbose(&format!("Removing sysroot {} to avoid caching bugs", sysroot.display())); + let _ = fs::remove_dir_all(&sysroot); + t!(fs::create_dir_all(&sysroot)); + + // In some cases(see https://github.com/rust-lang/rust/issues/109314), when the stage0 + // compiler relies on more recent version of LLVM than the beta compiler, it may not + // be able to locate the correct LLVM in the sysroot. This situation typically occurs + // when we upgrade LLVM version while the beta compiler continues to use an older version. + // + // Make sure to add the correct version of LLVM into the stage0 sysroot. + if compiler.stage == 0 { + dist::maybe_install_llvm_target(builder, compiler.host, &sysroot); + } + + // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. + if builder.download_rustc() && compiler.stage != 0 { + assert_eq!( + builder.config.build, compiler.host, + "Cross-compiling is not yet supported with `download-rustc`", + ); + + // #102002, cleanup old toolchain folders when using download-rustc so people don't use them by accident. + for stage in 0..=2 { + if stage != compiler.stage { + let dir = sysroot_dir(stage); + if !dir.ends_with("ci-rustc-sysroot") { + let _ = fs::remove_dir_all(dir); + } + } + } + + // Copy the compiler into the correct sysroot. + // NOTE(#108767): We intentionally don't copy `rustc-dev` artifacts until they're requested with `builder.ensure(Rustc)`. + // This fixes an issue where we'd have multiple copies of libc in the sysroot with no way to tell which to load. + // There are a few quirks of bootstrap that interact to make this reliable: + // 1. The order `Step`s are run is hard-coded in `builder.rs` and not configurable. This + // avoids e.g. reordering `test::UiFulldeps` before `test::Ui` and causing the latter to + // fail because of duplicate metadata. + // 2. The sysroot is deleted and recreated between each invocation, so running `x test + // ui-fulldeps && x test ui` can't cause failures. + let mut filtered_files = Vec::new(); + let mut add_filtered_files = |suffix, contents| { + for path in contents { + let path = Path::new(&path); + if path.parent().map_or(false, |parent| parent.ends_with(&suffix)) { + filtered_files.push(path.file_name().unwrap().to_owned()); + } + } + }; + let suffix = format!("lib/rustlib/{}/lib", compiler.host); + add_filtered_files(suffix.as_str(), builder.config.ci_rustc_dev_contents()); + // NOTE: we can't copy std eagerly because `stage2-test-sysroot` needs to have only the + // newly compiled std, not the downloaded std. + add_filtered_files("lib", builder.config.ci_rust_std_contents()); + + let filtered_extensions = [ + OsStr::new("rmeta"), + OsStr::new("rlib"), + // FIXME: this is wrong when compiler.host != build, but we don't support that today + OsStr::new(std::env::consts::DLL_EXTENSION), + ]; + let ci_rustc_dir = builder.config.ci_rustc_dir(); + builder.cp_filtered(&ci_rustc_dir, &sysroot, &|path| { + if path.extension().map_or(true, |ext| !filtered_extensions.contains(&ext)) { + return true; + } + if !path.parent().map_or(true, |p| p.ends_with(&suffix)) { + return true; + } + if !filtered_files.iter().all(|f| f != path.file_name().unwrap()) { + builder.verbose_than(1, &format!("ignoring {}", path.display())); + false + } else { + true + } + }); + } + + // Symlink the source root into the same location inside the sysroot, + // where `rust-src` component would go (`$sysroot/lib/rustlib/src/rust`), + // so that any tools relying on `rust-src` also work for local builds, + // and also for translating the virtual `/rustc/$hash` back to the real + // directory (for running tests with `rust.remap-debuginfo = true`). + let sysroot_lib_rustlib_src = sysroot.join("lib/rustlib/src"); + t!(fs::create_dir_all(&sysroot_lib_rustlib_src)); + let sysroot_lib_rustlib_src_rust = sysroot_lib_rustlib_src.join("rust"); + if let Err(e) = symlink_dir(&builder.config, &builder.src, &sysroot_lib_rustlib_src_rust) { + eprintln!( + "WARNING: creating symbolic link `{}` to `{}` failed with {}", + sysroot_lib_rustlib_src_rust.display(), + builder.src.display(), + e, + ); + if builder.config.rust_remap_debuginfo { + eprintln!( + "WARNING: some `tests/ui` tests will fail when lacking `{}`", + sysroot_lib_rustlib_src_rust.display(), + ); + } + } + // Same for the rustc-src component. + let sysroot_lib_rustlib_rustcsrc = sysroot.join("lib/rustlib/rustc-src"); + t!(fs::create_dir_all(&sysroot_lib_rustlib_rustcsrc)); + let sysroot_lib_rustlib_rustcsrc_rust = sysroot_lib_rustlib_rustcsrc.join("rust"); + if let Err(e) = + symlink_dir(&builder.config, &builder.src, &sysroot_lib_rustlib_rustcsrc_rust) + { + eprintln!( + "WARNING: creating symbolic link `{}` to `{}` failed with {}", + sysroot_lib_rustlib_rustcsrc_rust.display(), + builder.src.display(), + e, + ); + } + + INTERNER.intern_path(sysroot) + } +} + +#[derive(Debug, Copy, PartialOrd, Ord, Clone, PartialEq, Eq, Hash)] +pub struct Assemble { + /// The compiler which we will produce in this step. Assemble itself will + /// take care of ensuring that the necessary prerequisites to do so exist, + /// that is, this target can be a stage2 compiler and Assemble will build + /// previous stages for you. + pub target_compiler: Compiler, +} + +impl Step for Assemble { + type Output = Compiler; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("compiler/rustc").path("compiler") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Assemble { + target_compiler: run.builder.compiler(run.builder.top_stage + 1, run.target), + }); + } + + /// Prepare a new compiler from the artifacts in `stage` + /// + /// This will assemble a compiler in `build/$host/stage$stage`. The compiler + /// must have been previously produced by the `stage - 1` builder.build + /// compiler. + fn run(self, builder: &Builder<'_>) -> Compiler { + let target_compiler = self.target_compiler; + + if target_compiler.stage == 0 { + assert_eq!( + builder.config.build, target_compiler.host, + "Cannot obtain compiler for non-native build triple at stage 0" + ); + // The stage 0 compiler for the build triple is always pre-built. + return target_compiler; + } + + // Get the compiler that we'll use to bootstrap ourselves. + // + // Note that this is where the recursive nature of the bootstrap + // happens, as this will request the previous stage's compiler on + // downwards to stage 0. + // + // Also note that we're building a compiler for the host platform. We + // only assume that we can run `build` artifacts, which means that to + // produce some other architecture compiler we need to start from + // `build` to get there. + // + // FIXME: It may be faster if we build just a stage 1 compiler and then + // use that to bootstrap this compiler forward. + let build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build); + + // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. + if builder.download_rustc() { + let sysroot = + builder.ensure(Sysroot { compiler: target_compiler, force_recompile: false }); + // Ensure that `libLLVM.so` ends up in the newly created target directory, + // so that tools using `rustc_private` can use it. + dist::maybe_install_llvm_target(builder, target_compiler.host, &sysroot); + // Lower stages use `ci-rustc-sysroot`, not stageN + if target_compiler.stage == builder.top_stage { + builder.info(&format!("Creating a sysroot for stage{stage} compiler (use `rustup toolchain link 'name' build/host/stage{stage}`)", stage=target_compiler.stage)); + } + return target_compiler; + } + + // Build the libraries for this compiler to link to (i.e., the libraries + // it uses at runtime). NOTE: Crates the target compiler compiles don't + // link to these. (FIXME: Is that correct? It seems to be correct most + // of the time but I think we do link to these for stage2/bin compilers + // when not performing a full bootstrap). + builder.ensure(Rustc::new(build_compiler, target_compiler.host)); + + // FIXME: For now patch over problems noted in #90244 by early returning here, even though + // we've not properly assembled the target sysroot. A full fix is pending further investigation, + // for now full bootstrap usage is rare enough that this is OK. + if target_compiler.stage >= 3 && !builder.config.full_bootstrap { + return target_compiler; + } + + for &backend in builder.config.rust_codegen_backends.iter() { + if backend == "llvm" { + continue; // Already built as part of rustc + } + + builder.ensure(CodegenBackend { + compiler: build_compiler, + target: target_compiler.host, + backend, + }); + } + + let lld_install = if builder.config.lld_enabled { + Some(builder.ensure(llvm::Lld { target: target_compiler.host })) + } else { + None + }; + + let stage = target_compiler.stage; + let host = target_compiler.host; + let (host_info, dir_name) = if build_compiler.host == host { + ("".into(), "host".into()) + } else { + (format!(" ({host})"), host.to_string()) + }; + // NOTE: "Creating a sysroot" is somewhat inconsistent with our internal terminology, since + // sysroots can temporarily be empty until we put the compiler inside. However, + // `ensure(Sysroot)` isn't really something that's user facing, so there shouldn't be any + // ambiguity. + let msg = format!( + "Creating a sysroot for stage{stage} compiler{host_info} (use `rustup toolchain link 'name' build/{dir_name}/stage{stage}`)" + ); + builder.info(&msg); + + // Link in all dylibs to the libdir + let stamp = librustc_stamp(builder, build_compiler, target_compiler.host); + let proc_macros = builder + .read_stamp_file(&stamp) + .into_iter() + .filter_map(|(path, dependency_type)| { + if dependency_type == DependencyType::Host { + Some(path.file_name().unwrap().to_owned().into_string().unwrap()) + } else { + None + } + }) + .collect::>(); + + let sysroot = builder.sysroot(target_compiler); + let rustc_libdir = builder.rustc_libdir(target_compiler); + t!(fs::create_dir_all(&rustc_libdir)); + let src_libdir = builder.sysroot_libdir(build_compiler, host); + for f in builder.read_dir(&src_libdir) { + let filename = f.file_name().into_string().unwrap(); + if (is_dylib(&filename) || is_debug_info(&filename)) && !proc_macros.contains(&filename) + { + builder.copy(&f.path(), &rustc_libdir.join(&filename)); + } + } + + copy_codegen_backends_to_sysroot(builder, build_compiler, target_compiler); + + // We prepend this bin directory to the user PATH when linking Rust binaries. To + // avoid shadowing the system LLD we rename the LLD we provide to `rust-lld`. + let libdir = builder.sysroot_libdir(target_compiler, target_compiler.host); + let libdir_bin = libdir.parent().unwrap().join("bin"); + t!(fs::create_dir_all(&libdir_bin)); + if let Some(lld_install) = lld_install { + let src_exe = exe("lld", target_compiler.host); + let dst_exe = exe("rust-lld", target_compiler.host); + builder.copy(&lld_install.join("bin").join(&src_exe), &libdir_bin.join(&dst_exe)); + let self_contained_lld_dir = libdir_bin.join("gcc-ld"); + t!(fs::create_dir(&self_contained_lld_dir)); + let lld_wrapper_exe = builder.ensure(crate::core::build_steps::tool::LldWrapper { + compiler: build_compiler, + target: target_compiler.host, + }); + for name in crate::LLD_FILE_NAMES { + builder.copy( + &lld_wrapper_exe, + &self_contained_lld_dir.join(exe(name, target_compiler.host)), + ); + } + } + + if builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) { + let llvm::LlvmResult { llvm_config, .. } = + builder.ensure(llvm::Llvm { target: target_compiler.host }); + if !builder.config.dry_run() { + let llvm_bin_dir = output(Command::new(llvm_config).arg("--bindir")); + let llvm_bin_dir = Path::new(llvm_bin_dir.trim()); + + // Since we've already built the LLVM tools, install them to the sysroot. + // This is the equivalent of installing the `llvm-tools-preview` component via + // rustup, and lets developers use a locally built toolchain to + // build projects that expect llvm tools to be present in the sysroot + // (e.g. the `bootimage` crate). + for tool in LLVM_TOOLS { + let tool_exe = exe(tool, target_compiler.host); + let src_path = llvm_bin_dir.join(&tool_exe); + // When using `download-ci-llvm`, some of the tools + // may not exist, so skip trying to copy them. + if src_path.exists() { + builder.copy(&src_path, &libdir_bin.join(&tool_exe)); + } + } + } + } + + // Ensure that `libLLVM.so` ends up in the newly build compiler directory, + // so that it can be found when the newly built `rustc` is run. + dist::maybe_install_llvm_runtime(builder, target_compiler.host, &sysroot); + dist::maybe_install_llvm_target(builder, target_compiler.host, &sysroot); + + // Link the compiler binary itself into place + let out_dir = builder.cargo_out(build_compiler, Mode::Rustc, host); + let rustc = out_dir.join(exe("rustc-main", host)); + let bindir = sysroot.join("bin"); + t!(fs::create_dir_all(&bindir)); + let compiler = builder.rustc(target_compiler); + builder.copy(&rustc, &compiler); + + target_compiler + } +} + +/// Link some files into a rustc sysroot. +/// +/// For a particular stage this will link the file listed in `stamp` into the +/// `sysroot_dst` provided. +pub fn add_to_sysroot( + builder: &Builder<'_>, + sysroot_dst: &Path, + sysroot_host_dst: &Path, + stamp: &Path, +) { + let self_contained_dst = &sysroot_dst.join("self-contained"); + t!(fs::create_dir_all(&sysroot_dst)); + t!(fs::create_dir_all(&sysroot_host_dst)); + t!(fs::create_dir_all(&self_contained_dst)); + for (path, dependency_type) in builder.read_stamp_file(stamp) { + let dst = match dependency_type { + DependencyType::Host => sysroot_host_dst, + DependencyType::Target => sysroot_dst, + DependencyType::TargetSelfContained => self_contained_dst, + }; + builder.copy(&path, &dst.join(path.file_name().unwrap())); + } +} + +pub fn run_cargo( + builder: &Builder<'_>, + cargo: Cargo, + tail_args: Vec, + stamp: &Path, + additional_target_deps: Vec<(PathBuf, DependencyType)>, + is_check: bool, + rlib_only_metadata: bool, +) -> Vec { + if builder.config.dry_run() { + return Vec::new(); + } + + // `target_root_dir` looks like $dir/$target/release + let target_root_dir = stamp.parent().unwrap(); + // `target_deps_dir` looks like $dir/$target/release/deps + let target_deps_dir = target_root_dir.join("deps"); + // `host_root_dir` looks like $dir/release + let host_root_dir = target_root_dir + .parent() + .unwrap() // chop off `release` + .parent() + .unwrap() // chop off `$target` + .join(target_root_dir.file_name().unwrap()); + + // Spawn Cargo slurping up its JSON output. We'll start building up the + // `deps` array of all files it generated along with a `toplevel` array of + // files we need to probe for later. + let mut deps = Vec::new(); + let mut toplevel = Vec::new(); + let ok = stream_cargo(builder, cargo, tail_args, &mut |msg| { + let (filenames, crate_types) = match msg { + CargoMessage::CompilerArtifact { + filenames, + target: CargoTarget { crate_types }, + .. + } => (filenames, crate_types), + _ => return, + }; + for filename in filenames { + // Skip files like executables + let mut keep = false; + if filename.ends_with(".lib") + || filename.ends_with(".a") + || is_debug_info(&filename) + || is_dylib(&filename) + { + // Always keep native libraries, rust dylibs and debuginfo + keep = true; + } + if is_check && filename.ends_with(".rmeta") { + // During check builds we need to keep crate metadata + keep = true; + } else if rlib_only_metadata { + if filename.contains("jemalloc_sys") + || filename.contains("rustc_smir") + || filename.contains("stable_mir") + { + // jemalloc_sys and rustc_smir are not linked into librustc_driver.so, + // so we need to distribute them as rlib to be able to use them. + keep |= filename.ends_with(".rlib"); + } else { + // Distribute the rest of the rustc crates as rmeta files only to reduce + // the tarball sizes by about 50%. The object files are linked into + // librustc_driver.so, so it is still possible to link against them. + keep |= filename.ends_with(".rmeta"); + } + } else { + // In all other cases keep all rlibs + keep |= filename.ends_with(".rlib"); + } + + if !keep { + continue; + } + + let filename = Path::new(&*filename); + + // If this was an output file in the "host dir" we don't actually + // worry about it, it's not relevant for us + if filename.starts_with(&host_root_dir) { + // Unless it's a proc macro used in the compiler + if crate_types.iter().any(|t| t == "proc-macro") { + deps.push((filename.to_path_buf(), DependencyType::Host)); + } + continue; + } + + // If this was output in the `deps` dir then this is a precise file + // name (hash included) so we start tracking it. + if filename.starts_with(&target_deps_dir) { + deps.push((filename.to_path_buf(), DependencyType::Target)); + continue; + } + + // Otherwise this was a "top level artifact" which right now doesn't + // have a hash in the name, but there's a version of this file in + // the `deps` folder which *does* have a hash in the name. That's + // the one we'll want to we'll probe for it later. + // + // We do not use `Path::file_stem` or `Path::extension` here, + // because some generated files may have multiple extensions e.g. + // `std-.dll.lib` on Windows. The aforementioned methods only + // split the file name by the last extension (`.lib`) while we need + // to split by all extensions (`.dll.lib`). + let expected_len = t!(filename.metadata()).len(); + let filename = filename.file_name().unwrap().to_str().unwrap(); + let mut parts = filename.splitn(2, '.'); + let file_stem = parts.next().unwrap().to_owned(); + let extension = parts.next().unwrap().to_owned(); + + toplevel.push((file_stem, extension, expected_len)); + } + }); + + if !ok { + crate::exit!(1); + } + + // Ok now we need to actually find all the files listed in `toplevel`. We've + // got a list of prefix/extensions and we basically just need to find the + // most recent file in the `deps` folder corresponding to each one. + let contents = t!(target_deps_dir.read_dir()) + .map(|e| t!(e)) + .map(|e| (e.path(), e.file_name().into_string().unwrap(), t!(e.metadata()))) + .collect::>(); + for (prefix, extension, expected_len) in toplevel { + let candidates = contents.iter().filter(|&&(_, ref filename, ref meta)| { + meta.len() == expected_len + && filename + .strip_prefix(&prefix[..]) + .map(|s| s.starts_with('-') && s.ends_with(&extension[..])) + .unwrap_or(false) + }); + let max = candidates.max_by_key(|&&(_, _, ref metadata)| { + metadata.modified().expect("mtime should be available on all relevant OSes") + }); + let path_to_add = match max { + Some(triple) => triple.0.to_str().unwrap(), + None => panic!("no output generated for {prefix:?} {extension:?}"), + }; + if is_dylib(path_to_add) { + let candidate = format!("{path_to_add}.lib"); + let candidate = PathBuf::from(candidate); + if candidate.exists() { + deps.push((candidate, DependencyType::Target)); + } + } + deps.push((path_to_add.into(), DependencyType::Target)); + } + + deps.extend(additional_target_deps); + deps.sort(); + let mut new_contents = Vec::new(); + for (dep, dependency_type) in deps.iter() { + new_contents.extend(match *dependency_type { + DependencyType::Host => b"h", + DependencyType::Target => b"t", + DependencyType::TargetSelfContained => b"s", + }); + new_contents.extend(dep.to_str().unwrap().as_bytes()); + new_contents.extend(b"\0"); + } + t!(fs::write(&stamp, &new_contents)); + deps.into_iter().map(|(d, _)| d).collect() +} + +pub fn stream_cargo( + builder: &Builder<'_>, + cargo: Cargo, + tail_args: Vec, + cb: &mut dyn FnMut(CargoMessage<'_>), +) -> bool { + let mut cargo = Command::from(cargo); + if builder.config.dry_run() { + return true; + } + // Instruct Cargo to give us json messages on stdout, critically leaving + // stderr as piped so we can get those pretty colors. + let mut message_format = if builder.config.json_output { + String::from("json") + } else { + String::from("json-render-diagnostics") + }; + if let Some(s) = &builder.config.rustc_error_format { + message_format.push_str(",json-diagnostic-"); + message_format.push_str(s); + } + cargo.arg("--message-format").arg(message_format).stdout(Stdio::piped()); + + for arg in tail_args { + cargo.arg(arg); + } + + builder.verbose(&format!("running: {cargo:?}")); + let mut child = match cargo.spawn() { + Ok(child) => child, + Err(e) => panic!("failed to execute command: {cargo:?}\nERROR: {e}"), + }; + + // Spawn Cargo slurping up its JSON output. We'll start building up the + // `deps` array of all files it generated along with a `toplevel` array of + // files we need to probe for later. + let stdout = BufReader::new(child.stdout.take().unwrap()); + for line in stdout.lines() { + let line = t!(line); + match serde_json::from_str::>(&line) { + Ok(msg) => { + if builder.config.json_output { + // Forward JSON to stdout. + println!("{line}"); + } + cb(msg) + } + // If this was informational, just print it out and continue + Err(_) => println!("{line}"), + } + } + + // Make sure Cargo actually succeeded after we read all of its stdout. + let status = t!(child.wait()); + if builder.is_verbose() && !status.success() { + eprintln!( + "command did not execute successfully: {cargo:?}\n\ + expected success, got: {status}" + ); + } + status.success() +} + +#[derive(Deserialize)] +pub struct CargoTarget<'a> { + crate_types: Vec>, +} + +#[derive(Deserialize)] +#[serde(tag = "reason", rename_all = "kebab-case")] +pub enum CargoMessage<'a> { + CompilerArtifact { + package_id: Cow<'a, str>, + features: Vec>, + filenames: Vec>, + target: CargoTarget<'a>, + }, + BuildScriptExecuted { + package_id: Cow<'a, str>, + }, + BuildFinished { + success: bool, + }, +} + +pub fn strip_debug(builder: &Builder<'_>, target: TargetSelection, path: &Path) { + // FIXME: to make things simpler for now, limit this to the host and target where we know + // `strip -g` is both available and will fix the issue, i.e. on a x64 linux host that is not + // cross-compiling. Expand this to other appropriate targets in the future. + if target != "x86_64-unknown-linux-gnu" || target != builder.config.build || !path.exists() { + return; + } + + let previous_mtime = FileTime::from_last_modification_time(&path.metadata().unwrap()); + // NOTE: `output` will propagate any errors here. + output(Command::new("strip").arg("--strip-debug").arg(path)); + + // After running `strip`, we have to set the file modification time to what it was before, + // otherwise we risk Cargo invalidating its fingerprint and rebuilding the world next time + // bootstrap is invoked. + // + // An example of this is if we run this on librustc_driver.so. In the first invocation: + // - Cargo will build librustc_driver.so (mtime of 1) + // - Cargo will build rustc-main (mtime of 2) + // - Bootstrap will strip librustc_driver.so (changing the mtime to 3). + // + // In the second invocation of bootstrap, Cargo will see that the mtime of librustc_driver.so + // is greater than the mtime of rustc-main, and will rebuild rustc-main. That will then cause + // everything else (standard library, future stages...) to be rebuilt. + t!(filetime::set_file_mtime(path, previous_mtime)); +} diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs new file mode 100644 index 000000000..c485481b9 --- /dev/null +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -0,0 +1,2366 @@ +//! Implementation of the various distribution aspects of the compiler. +//! +//! This module is responsible for creating tarballs of the standard library, +//! compiler, and documentation. This ends up being what we distribute to +//! everyone as well. +//! +//! No tarball is actually created literally in this file, but rather we shell +//! out to `rust-installer` still. This may one day be replaced with bits and +//! pieces of `rustup.rs`! + +use std::collections::HashSet; +use std::env; +use std::ffi::OsStr; +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use object::read::archive::ArchiveFile; +use object::BinaryFormat; + +use crate::core::build_steps::compile; +use crate::core::build_steps::doc::DocumentationFormat; +use crate::core::build_steps::llvm; +use crate::core::build_steps::tool::{self, Tool}; +use crate::core::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; +use crate::core::config::TargetSelection; +use crate::utils::cache::{Interned, INTERNER}; +use crate::utils::channel; +use crate::utils::helpers::{exe, is_dylib, output, t, target_supports_cranelift_backend, timeit}; +use crate::utils::tarball::{GeneratedTarball, OverlayKind, Tarball}; +use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS}; + +pub fn pkgname(builder: &Builder<'_>, component: &str) -> String { + format!("{}-{}", component, builder.rust_package_vers()) +} + +pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf { + builder.out.join("dist") +} + +pub fn tmpdir(builder: &Builder<'_>) -> PathBuf { + builder.out.join("tmp/dist") +} + +fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool { + if !builder.config.extended { + return false; + } + builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool)) +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Docs { + pub host: TargetSelection, +} + +impl Step for Docs { + type Output = Option; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = run.builder.config.docs; + run.alias("rust-docs").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Docs { host: run.target }); + } + + /// Builds the `rust-docs` installer component. + fn run(self, builder: &Builder<'_>) -> Option { + let host = self.host; + builder.default_doc(&[]); + + let dest = "share/doc/rust/html"; + + let mut tarball = Tarball::new(builder, "rust-docs", &host.triple); + tarball.set_product_name("Rust Documentation"); + tarball.add_bulk_dir(&builder.doc_out(host), dest); + tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644); + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct JsonDocs { + pub host: TargetSelection, +} + +impl Step for JsonDocs { + type Output = Option; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = run.builder.config.docs; + run.alias("rust-docs-json").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(JsonDocs { host: run.target }); + } + + /// Builds the `rust-docs-json` installer component. + fn run(self, builder: &Builder<'_>) -> Option { + let host = self.host; + builder.ensure(crate::core::build_steps::doc::Std::new( + builder.top_stage, + host, + builder, + DocumentationFormat::JSON, + )); + + let dest = "share/doc/rust/json"; + + let mut tarball = Tarball::new(builder, "rust-docs-json", &host.triple); + tarball.set_product_name("Rust Documentation In JSON Format"); + tarball.is_preview(true); + tarball.add_bulk_dir(&builder.json_doc_out(host), dest); + Some(tarball.generate()) + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustcDocs { + pub host: TargetSelection, +} + +impl Step for RustcDocs { + type Output = Option; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.alias("rustc-docs").default_condition(builder.config.compiler_docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustcDocs { host: run.target }); + } + + /// Builds the `rustc-docs` installer component. + fn run(self, builder: &Builder<'_>) -> Option { + let host = self.host; + builder.default_doc(&[]); + + let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple); + tarball.set_product_name("Rustc Documentation"); + tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc"); + Some(tarball.generate()) + } +} + +fn find_files(files: &[&str], path: &[PathBuf]) -> Vec { + let mut found = Vec::with_capacity(files.len()); + + for file in files { + let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists()); + + if let Some(file_path) = file_path { + found.push(file_path); + } else { + panic!("Could not find '{file}' in {path:?}"); + } + } + + found +} + +fn make_win_dist( + rust_root: &Path, + plat_root: &Path, + target: TargetSelection, + builder: &Builder<'_>, +) { + if builder.config.dry_run() { + return; + } + + //Ask gcc where it keeps its stuff + let mut cmd = Command::new(builder.cc(target)); + cmd.arg("-print-search-dirs"); + let gcc_out = output(&mut cmd); + + let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect(); + let mut lib_path = Vec::new(); + + for line in gcc_out.lines() { + let idx = line.find(':').unwrap(); + let key = &line[..idx]; + let trim_chars: &[_] = &[' ', '=']; + let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars)); + + if key == "programs" { + bin_path.extend(value); + } else if key == "libraries" { + lib_path.extend(value); + } + } + + let compiler = if target == "i686-pc-windows-gnu" { + "i686-w64-mingw32-gcc.exe" + } else if target == "x86_64-pc-windows-gnu" { + "x86_64-w64-mingw32-gcc.exe" + } else { + "gcc.exe" + }; + let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"]; + let mut rustc_dlls = vec!["libwinpthread-1.dll"]; + if target.starts_with("i686-") { + rustc_dlls.push("libgcc_s_dw2-1.dll"); + } else { + rustc_dlls.push("libgcc_s_seh-1.dll"); + } + + // Libraries necessary to link the windows-gnu toolchains. + // System libraries will be preferred if they are available (see #67429). + let target_libs = [ + //MinGW libs + "libgcc.a", + "libgcc_eh.a", + "libgcc_s.a", + "libm.a", + "libmingw32.a", + "libmingwex.a", + "libstdc++.a", + "libiconv.a", + "libmoldname.a", + "libpthread.a", + //Windows import libs + //This should contain only the set of libraries necessary to link the standard library. + "libadvapi32.a", + "libbcrypt.a", + "libcomctl32.a", + "libcomdlg32.a", + "libcredui.a", + "libcrypt32.a", + "libdbghelp.a", + "libgdi32.a", + "libimagehlp.a", + "libiphlpapi.a", + "libkernel32.a", + "libmsimg32.a", + "libmsvcrt.a", + "libntdll.a", + "libodbc32.a", + "libole32.a", + "liboleaut32.a", + "libopengl32.a", + "libpsapi.a", + "librpcrt4.a", + "libsecur32.a", + "libsetupapi.a", + "libshell32.a", + "libsynchronization.a", + "libuser32.a", + "libuserenv.a", + "libuuid.a", + "libwinhttp.a", + "libwinmm.a", + "libwinspool.a", + "libws2_32.a", + "libwsock32.a", + ]; + + //Find mingw artifacts we want to bundle + let target_tools = find_files(&target_tools, &bin_path); + let rustc_dlls = find_files(&rustc_dlls, &bin_path); + let target_libs = find_files(&target_libs, &lib_path); + + // Copy runtime dlls next to rustc.exe + let dist_bin_dir = rust_root.join("bin/"); + fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed"); + for src in rustc_dlls { + builder.copy_to_folder(&src, &dist_bin_dir); + } + + //Copy platform tools to platform-specific bin directory + let target_bin_dir = plat_root + .join("lib") + .join("rustlib") + .join(target.triple) + .join("bin") + .join("self-contained"); + fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed"); + for src in target_tools { + builder.copy_to_folder(&src, &target_bin_dir); + } + + // Warn windows-gnu users that the bundled GCC cannot compile C files + builder.create( + &target_bin_dir.join("GCC-WARNING.txt"), + "gcc.exe contained in this folder cannot be used for compiling C files - it is only \ + used as a linker. In order to be able to compile projects containing C code use \ + the GCC provided by MinGW or Cygwin.", + ); + + //Copy platform libs to platform-specific lib directory + let target_lib_dir = plat_root + .join("lib") + .join("rustlib") + .join(target.triple) + .join("lib") + .join("self-contained"); + fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed"); + for src in target_libs { + builder.copy_to_folder(&src, &target_lib_dir); + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Mingw { + pub host: TargetSelection, +} + +impl Step for Mingw { + type Output = Option; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("rust-mingw") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Mingw { host: run.target }); + } + + /// Builds the `rust-mingw` installer component. + /// + /// This contains all the bits and pieces to run the MinGW Windows targets + /// without any extra installed software (e.g., we bundle gcc, libraries, etc). + fn run(self, builder: &Builder<'_>) -> Option { + let host = self.host; + if !host.ends_with("pc-windows-gnu") || !builder.config.dist_include_mingw_linker { + return None; + } + + let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple); + tarball.set_product_name("Rust MinGW"); + + // The first argument is a "temporary directory" which is just + // thrown away (this contains the runtime DLLs included in the rustc package + // above) and the second argument is where to place all the MinGW components + // (which is what we want). + make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder); + + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Rustc { + pub compiler: Compiler, +} + +impl Step for Rustc { + type Output = GeneratedTarball; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("rustc") + } + + fn make_run(run: RunConfig<'_>) { + run.builder + .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) }); + } + + /// Creates the `rustc` installer component. + fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + let compiler = self.compiler; + let host = self.compiler.host; + + let tarball = Tarball::new(builder, "rustc", &host.triple); + + // Prepare the rustc "image", what will actually end up getting installed + prepare_image(builder, compiler, tarball.image_dir()); + + // On MinGW we've got a few runtime DLL dependencies that we need to + // include. The first argument to this script is where to put these DLLs + // (the image we're creating), and the second argument is a junk directory + // to ignore all other MinGW stuff the script creates. + // + // On 32-bit MinGW we're always including a DLL which needs some extra + // licenses to distribute. On 64-bit MinGW we don't actually distribute + // anything requiring us to distribute a license, but it's likely the + // install will *also* include the rust-mingw package, which also needs + // licenses, so to be safe we just include it here in all MinGW packages. + if host.ends_with("pc-windows-gnu") && builder.config.dist_include_mingw_linker { + make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder); + tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc"); + } + + return tarball.generate(); + + fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) { + let host = compiler.host; + let src = builder.sysroot(compiler); + + // Copy rustc/rustdoc binaries + t!(fs::create_dir_all(image.join("bin"))); + builder.cp_r(&src.join("bin"), &image.join("bin")); + + if builder + .config + .tools + .as_ref() + .map_or(true, |tools| tools.iter().any(|tool| tool == "rustdoc")) + { + let rustdoc = builder.rustdoc(compiler); + builder.install(&rustdoc, &image.join("bin"), 0o755); + } + + if let Some(ra_proc_macro_srv) = builder.ensure_if_default( + tool::RustAnalyzerProcMacroSrv { + compiler: builder.compiler_for( + compiler.stage, + builder.config.build, + compiler.host, + ), + target: compiler.host, + }, + builder.kind, + ) { + builder.install(&ra_proc_macro_srv, &image.join("libexec"), 0o755); + } + + let libdir_relative = builder.libdir_relative(compiler); + + // Copy runtime DLLs needed by the compiler + if libdir_relative.to_str() != Some("bin") { + let libdir = builder.rustc_libdir(compiler); + for entry in builder.read_dir(&libdir) { + let name = entry.file_name(); + if let Some(s) = name.to_str() { + if is_dylib(s) { + // Don't use custom libdir here because ^lib/ will be resolved again + // with installer + builder.install(&entry.path(), &image.join("lib"), 0o644); + } + } + } + } + + // Copy libLLVM.so to the lib dir as well, if needed. While not + // technically needed by rustc itself it's needed by lots of other + // components like the llvm tools and LLD. LLD is included below and + // tools/LLDB come later, so let's just throw it in the rustc + // component for now. + maybe_install_llvm_runtime(builder, host, image); + + let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin"); + t!(fs::create_dir_all(&dst_dir)); + + // Copy over lld if it's there + if builder.config.lld_enabled { + let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin"); + let rust_lld = exe("rust-lld", compiler.host); + builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld)); + let self_contained_lld_src_dir = src_dir.join("gcc-ld"); + let self_contained_lld_dst_dir = dst_dir.join("gcc-ld"); + t!(fs::create_dir(&self_contained_lld_dst_dir)); + for name in crate::LLD_FILE_NAMES { + let exe_name = exe(name, compiler.host); + builder.copy( + &self_contained_lld_src_dir.join(&exe_name), + &self_contained_lld_dst_dir.join(&exe_name), + ); + } + } + + // Man pages + t!(fs::create_dir_all(image.join("share/man/man1"))); + let man_src = builder.src.join("src/doc/man"); + let man_dst = image.join("share/man/man1"); + + // don't use our `bootstrap::{copy, cp_r}`, because those try + // to hardlink, and we don't want to edit the source templates + for file_entry in builder.read_dir(&man_src) { + let page_src = file_entry.path(); + let page_dst = man_dst.join(file_entry.file_name()); + let src_text = t!(std::fs::read_to_string(&page_src)); + let new_text = src_text.replace("", &builder.version); + t!(std::fs::write(&page_dst, &new_text)); + t!(fs::copy(&page_src, &page_dst)); + } + + // Debugger scripts + builder + .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host }); + + // Misc license info + let cp = |file: &str| { + builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644); + }; + cp("COPYRIGHT"); + cp("LICENSE-APACHE"); + cp("LICENSE-MIT"); + cp("README.md"); + } + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct DebuggerScripts { + pub sysroot: Interned, + pub host: TargetSelection, +} + +impl Step for DebuggerScripts { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Copies debugger scripts for `target` into the `sysroot` specified. + fn run(self, builder: &Builder<'_>) { + let host = self.host; + let sysroot = self.sysroot; + let dst = sysroot.join("lib/rustlib/etc"); + t!(fs::create_dir_all(&dst)); + let cp_debugger_script = |file: &str| { + builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644); + }; + if host.contains("windows-msvc") { + // windbg debugger scripts + builder.install( + &builder.src.join("src/etc/rust-windbg.cmd"), + &sysroot.join("bin"), + 0o755, + ); + + cp_debugger_script("natvis/intrinsic.natvis"); + cp_debugger_script("natvis/liballoc.natvis"); + cp_debugger_script("natvis/libcore.natvis"); + cp_debugger_script("natvis/libstd.natvis"); + } else { + cp_debugger_script("rust_types.py"); + + // gdb debugger scripts + builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755); + builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755); + + cp_debugger_script("gdb_load_rust_pretty_printers.py"); + cp_debugger_script("gdb_lookup.py"); + cp_debugger_script("gdb_providers.py"); + + // lldb debugger scripts + builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755); + + cp_debugger_script("lldb_lookup.py"); + cp_debugger_script("lldb_providers.py"); + cp_debugger_script("lldb_commands") + } + } +} + +fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool { + // The only true set of target libraries came from the build triple, so + // let's reduce redundant work by only producing archives from that host. + if compiler.host != builder.config.build { + builder.info("\tskipping, not a build host"); + true + } else { + false + } +} + +/// Check that all objects in rlibs for UEFI targets are COFF. This +/// ensures that the C compiler isn't producing ELF objects, which would +/// not link correctly with the COFF objects. +fn verify_uefi_rlib_format(builder: &Builder<'_>, target: TargetSelection, stamp: &Path) { + if !target.ends_with("-uefi") { + return; + } + + for (path, _) in builder.read_stamp_file(stamp) { + if path.extension() != Some(OsStr::new("rlib")) { + continue; + } + + let data = t!(fs::read(&path)); + let data = data.as_slice(); + let archive = t!(ArchiveFile::parse(data)); + for member in archive.members() { + let member = t!(member); + let member_data = t!(member.data(data)); + + let is_coff = match object::File::parse(member_data) { + Ok(member_file) => member_file.format() == BinaryFormat::Coff, + Err(_) => false, + }; + + if !is_coff { + let member_name = String::from_utf8_lossy(member.name()); + panic!("member {} in {} is not COFF", member_name, path.display()); + } + } + } +} + +/// Copy stamped files into an image's `target/lib` directory. +fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) { + let dst = image.join("lib/rustlib").join(target.triple).join("lib"); + let self_contained_dst = dst.join("self-contained"); + t!(fs::create_dir_all(&dst)); + t!(fs::create_dir_all(&self_contained_dst)); + for (path, dependency_type) in builder.read_stamp_file(stamp) { + if dependency_type == DependencyType::TargetSelfContained { + builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap())); + } else if dependency_type == DependencyType::Target || builder.config.build == target { + builder.copy(&path, &dst.join(path.file_name().unwrap())); + } + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Std { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Std { + type Output = Option; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("rust-std") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Std { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + + if skip_host_target_lib(builder, compiler) { + return None; + } + + builder.ensure(compile::Std::new(compiler, target)); + + let mut tarball = Tarball::new(builder, "rust-std", &target.triple); + tarball.include_target_in_component_name(true); + + let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); + let stamp = compile::libstd_stamp(builder, compiler_to_use, target); + verify_uefi_rlib_format(builder, target, &stamp); + copy_target_libs(builder, target, &tarball.image_dir(), &stamp); + + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustcDev { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for RustcDev { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("rustc-dev") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustcDev { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + if skip_host_target_lib(builder, compiler) { + return None; + } + + builder.ensure(compile::Rustc::new(compiler, target)); + + let tarball = Tarball::new(builder, "rustc-dev", &target.triple); + + let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); + let stamp = compile::librustc_stamp(builder, compiler_to_use, target); + copy_target_libs(builder, target, tarball.image_dir(), &stamp); + + let src_files = &["Cargo.lock"]; + // This is the reduced set of paths which will become the rustc-dev component + // (essentially the compiler crates and all of their path dependencies). + copy_src_dirs( + builder, + &builder.src, + &["compiler"], + &[], + &tarball.image_dir().join("lib/rustlib/rustc-src/rust"), + ); + for file in src_files { + tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644); + } + + Some(tarball.generate()) + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Analysis { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Analysis { + type Output = Option; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(&run.builder, "analysis"); + run.alias("rust-analysis").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Analysis { + // Find the actual compiler (handling the full bootstrap option) which + // produced the save-analysis data because that data isn't copied + // through the sysroot uplifting. + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + /// Creates a tarball of (degenerate) save-analysis metadata, if available. + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + if compiler.host != builder.config.build { + return None; + } + + let src = builder + .stage_out(compiler, Mode::Std) + .join(target.triple) + .join(builder.cargo_dir()) + .join("deps") + .join("save-analysis"); + + // Write a file indicating that this component has been removed. + t!(std::fs::create_dir_all(&src)); + let mut removed = src.clone(); + removed.push("removed.json"); + let mut f = t!(std::fs::File::create(removed)); + t!(write!(f, r#"{{ "warning": "The `rust-analysis` component has been removed." }}"#)); + + let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple); + tarball.include_target_in_component_name(true); + tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple)); + Some(tarball.generate()) + } +} + +/// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to +/// `dst_dir`. +fn copy_src_dirs( + builder: &Builder<'_>, + base: &Path, + src_dirs: &[&str], + exclude_dirs: &[&str], + dst_dir: &Path, +) { + fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool { + let spath = match path.to_str() { + Some(path) => path, + None => return false, + }; + if spath.ends_with('~') || spath.ends_with(".pyc") { + return false; + } + + const LLVM_PROJECTS: &[&str] = &[ + "llvm-project/clang", + "llvm-project\\clang", + "llvm-project/libunwind", + "llvm-project\\libunwind", + "llvm-project/lld", + "llvm-project\\lld", + "llvm-project/lldb", + "llvm-project\\lldb", + "llvm-project/llvm", + "llvm-project\\llvm", + "llvm-project/compiler-rt", + "llvm-project\\compiler-rt", + "llvm-project/cmake", + "llvm-project\\cmake", + "llvm-project/runtimes", + "llvm-project\\runtimes", + ]; + if spath.contains("llvm-project") + && !spath.ends_with("llvm-project") + && !LLVM_PROJECTS.iter().any(|path| spath.contains(path)) + { + return false; + } + + const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"]; + if LLVM_TEST.iter().any(|path| spath.contains(path)) + && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s")) + { + return false; + } + + let full_path = Path::new(dir).join(path); + if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) { + return false; + } + + let excludes = [ + "CVS", + "RCS", + "SCCS", + ".git", + ".gitignore", + ".gitmodules", + ".gitattributes", + ".cvsignore", + ".svn", + ".arch-ids", + "{arch}", + "=RELEASE-ID", + "=meta-update", + "=update", + ".bzr", + ".bzrignore", + ".bzrtags", + ".hg", + ".hgignore", + ".hgrags", + "_darcs", + ]; + !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s)) + } + + // Copy the directories using our filter + for item in src_dirs { + let dst = &dst_dir.join(item); + t!(fs::create_dir_all(dst)); + builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path)); + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Src; + +impl Step for Src { + /// The output path of the src installer tarball + type Output = GeneratedTarball; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("rust-src") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Src); + } + + /// Creates the `rust-src` installer component + fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + if !builder.config.dry_run() { + builder.update_submodule(&Path::new("src/llvm-project")); + } + + let tarball = Tarball::new_targetless(builder, "rust-src"); + + // A lot of tools expect the rust-src component to be entirely in this directory, so if you + // change that (e.g. by adding another directory `lib/rustlib/src/foo` or + // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions + // and fix them... + // + // NOTE: if you update the paths here, you also should update the "virtual" path + // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs` + let dst_src = tarball.image_dir().join("lib/rustlib/src/rust"); + + let src_files = ["Cargo.lock"]; + // This is the reduced set of paths which will become the rust-src component + // (essentially libstd and all of its path dependencies). + copy_src_dirs( + builder, + &builder.src, + &["library", "src/llvm-project/libunwind"], + &[ + // not needed and contains symlinks which rustup currently + // chokes on when unpacking. + "library/backtrace/crates", + // these are 30MB combined and aren't necessary for building + // the standard library. + "library/stdarch/Cargo.toml", + "library/stdarch/crates/stdarch-verify", + "library/stdarch/crates/intrinsic-test", + ], + &dst_src, + ); + for file in src_files.iter() { + builder.copy(&builder.src.join(file), &dst_src.join(file)); + } + + tarball.generate() + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct PlainSourceTarball; + +impl Step for PlainSourceTarball { + /// Produces the location of the tarball generated + type Output = GeneratedTarball; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.alias("rustc-src").default_condition(builder.config.rust_dist_src) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(PlainSourceTarball); + } + + /// Creates the plain source tarball + fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which + // means neither rustup nor rustup-toolchain-install-master know how to download it. + // It also contains symbolic links, unlike other any other dist tarball. + // It's used for distros building rustc from source in a pre-vendored environment. + let mut tarball = Tarball::new(builder, "rustc", "src"); + tarball.permit_symlinks(true); + let plain_dst_src = tarball.image_dir(); + + // This is the set of root paths which will become part of the source package + let src_files = [ + "COPYRIGHT", + "LICENSE-APACHE", + "LICENSE-MIT", + "CONTRIBUTING.md", + "README.md", + "RELEASES.md", + "configure", + "x.py", + "config.example.toml", + "Cargo.toml", + "Cargo.lock", + ".gitmodules", + ]; + let src_dirs = ["src", "compiler", "library", "tests"]; + + copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src); + + // Copy the files normally + for item in &src_files { + builder.copy(&builder.src.join(item), &plain_dst_src.join(item)); + } + + // Create the version file + builder.create(&plain_dst_src.join("version"), &builder.rust_version()); + if let Some(info) = builder.rust_info().info() { + channel::write_commit_hash_file(&plain_dst_src, &info.sha); + channel::write_commit_info_file(&plain_dst_src, info); + } + + // If we're building from git or tarball sources, we need to vendor + // a complete distribution. + if builder.rust_info().is_managed_git_subrepository() + || builder.rust_info().is_from_tarball() + { + if builder.rust_info().is_managed_git_subrepository() { + // Ensure we have the submodules checked out. + builder.update_submodule(Path::new("src/tools/cargo")); + } + + // Vendor all Cargo dependencies + let mut cmd = Command::new(&builder.initial_cargo); + cmd.arg("vendor") + .arg("--sync") + .arg(builder.src.join("./src/tools/cargo/Cargo.toml")) + .arg("--sync") + .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml")) + .arg("--sync") + .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml")) + .arg("--sync") + .arg(builder.src.join("./src/bootstrap/Cargo.toml")) + // Will read the libstd Cargo.toml + // which uses the unstable `public-dependency` feature. + .env("RUSTC_BOOTSTRAP", "1") + .current_dir(&plain_dst_src); + + let config = if !builder.config.dry_run() { + t!(String::from_utf8(t!(cmd.output()).stdout)) + } else { + String::new() + }; + + let cargo_config_dir = plain_dst_src.join(".cargo"); + builder.create_dir(&cargo_config_dir); + builder.create(&cargo_config_dir.join("config.toml"), &config); + } + + tarball.bare() + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Cargo { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Cargo { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(&run.builder, "cargo"); + run.alias("cargo").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Cargo { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + + let cargo = builder.ensure(tool::Cargo { compiler, target }); + let src = builder.src.join("src/tools/cargo"); + let etc = src.join("src/etc"); + + // Prepare the image directory + let mut tarball = Tarball::new(builder, "cargo", &target.triple); + tarball.set_overlay(OverlayKind::Cargo); + + tarball.add_file(&cargo, "bin", 0o755); + tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644); + tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo"); + tarball.add_dir(etc.join("man"), "share/man/man1"); + tarball.add_legal_and_readme_to("share/doc/cargo"); + + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Rls { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Rls { + type Output = Option; + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(&run.builder, "rls"); + run.alias("rls").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Rls { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + + let rls = builder + .ensure(tool::Rls { compiler, target, extra_features: Vec::new() }) + .expect("rls expected to build"); + + let mut tarball = Tarball::new(builder, "rls", &target.triple); + tarball.set_overlay(OverlayKind::RLS); + tarball.is_preview(true); + tarball.add_file(rls, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rls"); + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustAnalyzer { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for RustAnalyzer { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(&run.builder, "rust-analyzer"); + run.alias("rust-analyzer").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustAnalyzer { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + + let rust_analyzer = builder + .ensure(tool::RustAnalyzer { compiler, target }) + .expect("rust-analyzer always builds"); + + let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple); + tarball.set_overlay(OverlayKind::RustAnalyzer); + tarball.is_preview(true); + tarball.add_file(rust_analyzer, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rust-analyzer"); + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Clippy { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Clippy { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(&run.builder, "clippy"); + run.alias("clippy").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Clippy { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + + // Prepare the image directory + // We expect clippy to build, because we've exited this step above if tool + // state for clippy isn't testing. + let clippy = builder + .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() }) + .expect("clippy expected to build - essential tool"); + let cargoclippy = builder + .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() }) + .expect("clippy expected to build - essential tool"); + + let mut tarball = Tarball::new(builder, "clippy", &target.triple); + tarball.set_overlay(OverlayKind::Clippy); + tarball.is_preview(true); + tarball.add_file(clippy, "bin", 0o755); + tarball.add_file(cargoclippy, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/clippy"); + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Miri { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Miri { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(&run.builder, "miri"); + run.alias("miri").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Miri { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + // This prevents miri from being built for "dist" or "install" + // on the stable/beta channels. It is a nightly-only tool and should + // not be included. + if !builder.build.unstable_features() { + return None; + } + let compiler = self.compiler; + let target = self.target; + + let miri = builder.ensure(tool::Miri { compiler, target, extra_features: Vec::new() })?; + let cargomiri = + builder.ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })?; + + let mut tarball = Tarball::new(builder, "miri", &target.triple); + tarball.set_overlay(OverlayKind::Miri); + tarball.is_preview(true); + tarball.add_file(miri, "bin", 0o755); + tarball.add_file(cargomiri, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/miri"); + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct CodegenBackend { + pub compiler: Compiler, + pub backend: Interned, +} + +impl Step for CodegenBackend { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("compiler/rustc_codegen_cranelift") + } + + fn make_run(run: RunConfig<'_>) { + for &backend in &run.builder.config.rust_codegen_backends { + if backend == "llvm" { + continue; // Already built as part of rustc + } + + run.builder.ensure(CodegenBackend { + compiler: run.builder.compiler(run.builder.top_stage, run.target), + backend, + }); + } + } + + fn run(self, builder: &Builder<'_>) -> Option { + if builder.config.dry_run() { + return None; + } + + // This prevents rustc_codegen_cranelift from being built for "dist" + // or "install" on the stable/beta channels. It is not yet stable and + // should not be included. + if !builder.build.unstable_features() { + return None; + } + + if !builder.config.rust_codegen_backends.contains(&self.backend) { + return None; + } + + if self.backend == "cranelift" { + if !target_supports_cranelift_backend(self.compiler.host) { + builder.info("target not supported by rustc_codegen_cranelift. skipping"); + return None; + } + + if self.compiler.host.contains("windows") { + builder.info( + "dist currently disabled for windows by rustc_codegen_cranelift. skipping", + ); + return None; + } + } + + let compiler = self.compiler; + let backend = self.backend; + + let mut tarball = + Tarball::new(builder, &format!("rustc-codegen-{}", backend), &compiler.host.triple); + if backend == "cranelift" { + tarball.set_overlay(OverlayKind::RustcCodegenCranelift); + } else { + panic!("Unknown backend rustc_codegen_{}", backend); + } + tarball.is_preview(true); + tarball.add_legal_and_readme_to(format!("share/doc/rustc_codegen_{}", backend)); + + let src = builder.sysroot(compiler); + let backends_src = builder.sysroot_codegen_backends(compiler); + let backends_rel = backends_src + .strip_prefix(&src) + .unwrap() + .strip_prefix(builder.sysroot_libdir_relative(compiler)) + .unwrap(); + // Don't use custom libdir here because ^lib/ will be resolved again with installer + let backends_dst = PathBuf::from("lib").join(&backends_rel); + + let backend_name = format!("rustc_codegen_{}", backend); + let mut found_backend = false; + for backend in fs::read_dir(&backends_src).unwrap() { + let file_name = backend.unwrap().file_name(); + if file_name.to_str().unwrap().contains(&backend_name) { + tarball.add_file(backends_src.join(file_name), &backends_dst, 0o644); + found_backend = true; + } + } + assert!(found_backend); + + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Rustfmt { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Rustfmt { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(&run.builder, "rustfmt"); + run.alias("rustfmt").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Rustfmt { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + + let rustfmt = builder + .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() }) + .expect("rustfmt expected to build - essential tool"); + let cargofmt = builder + .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() }) + .expect("cargo fmt expected to build - essential tool"); + let mut tarball = Tarball::new(builder, "rustfmt", &target.triple); + tarball.set_overlay(OverlayKind::Rustfmt); + tarball.is_preview(true); + tarball.add_file(rustfmt, "bin", 0o755); + tarball.add_file(cargofmt, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rustfmt"); + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustDemangler { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for RustDemangler { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + // While other tools use `should_build_extended_tool` to decide whether to be run by + // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like + // the other ones or if `profiler = true`. Because we don't know the target at this stage + // we run the step by default when only `extended = true`, and decide whether to actually + // run it or not later. + let default = run.builder.config.extended; + run.alias("rust-demangler").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustDemangler { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + + // Only build this extended tool if explicitly included in `tools`, or if `profiler = true` + let condition = should_build_extended_tool(builder, "rust-demangler") + || builder.config.profiler_enabled(target); + if builder.config.extended && !condition { + return None; + } + + let rust_demangler = builder + .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() }) + .expect("rust-demangler expected to build - in-tree tool"); + + // Prepare the image directory + let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple); + tarball.set_overlay(OverlayKind::RustDemangler); + tarball.is_preview(true); + tarball.add_file(&rust_demangler, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rust-demangler"); + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Extended { + stage: u32, + host: TargetSelection, + target: TargetSelection, +} + +impl Step for Extended { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.alias("extended").default_condition(builder.config.extended) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Extended { + stage: run.builder.top_stage, + host: run.builder.config.build, + target: run.target, + }); + } + + /// Creates a combined installer for the specified target in the provided stage. + fn run(self, builder: &Builder<'_>) { + let target = self.target; + let stage = self.stage; + let compiler = builder.compiler_for(self.stage, self.host, self.target); + + builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target)); + + let mut tarballs = Vec::new(); + let mut built_tools = HashSet::new(); + macro_rules! add_component { + ($name:expr => $step:expr) => { + if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) { + tarballs.push(tarball); + built_tools.insert($name); + } + }; + } + + // When rust-std package split from rustc, we needed to ensure that during + // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering + // the std files during uninstall. To do this ensure that rustc comes + // before rust-std in the list below. + tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) })); + tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std")); + + if target.ends_with("windows-gnu") { + tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw")); + } + + add_component!("rust-docs" => Docs { host: target }); + add_component!("rust-json-docs" => JsonDocs { host: target }); + add_component!("rust-demangler"=> RustDemangler { compiler, target }); + add_component!("cargo" => Cargo { compiler, target }); + add_component!("rustfmt" => Rustfmt { compiler, target }); + add_component!("rls" => Rls { compiler, target }); + add_component!("rust-analyzer" => RustAnalyzer { compiler, target }); + add_component!("llvm-components" => LlvmTools { target }); + add_component!("clippy" => Clippy { compiler, target }); + add_component!("miri" => Miri { compiler, target }); + add_component!("analysis" => Analysis { compiler, target }); + add_component!("rustc-codegen-cranelift" => CodegenBackend { + compiler: builder.compiler(stage, target), + backend: INTERNER.intern_str("cranelift"), + }); + + let etc = builder.src.join("src/etc/installer"); + + // Avoid producing tarballs during a dry run. + if builder.config.dry_run() { + return; + } + + let tarball = Tarball::new(builder, "rust", &target.triple); + let generated = tarball.combine(&tarballs); + + let tmp = tmpdir(builder).join("combined-tarball"); + let work = generated.work_dir(); + + let mut license = String::new(); + license += &builder.read(&builder.src.join("COPYRIGHT")); + license += &builder.read(&builder.src.join("LICENSE-APACHE")); + license += &builder.read(&builder.src.join("LICENSE-MIT")); + license.push('\n'); + license.push('\n'); + + let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18"; + let mut rtf = rtf.to_string(); + rtf.push('\n'); + for line in license.lines() { + rtf.push_str(line); + rtf.push_str("\\line "); + } + rtf.push('}'); + + fn filter(contents: &str, marker: &str) -> String { + let start = format!("tool-{marker}-start"); + let end = format!("tool-{marker}-end"); + let mut lines = Vec::new(); + let mut omitted = false; + for line in contents.lines() { + if line.contains(&start) { + omitted = true; + } else if line.contains(&end) { + omitted = false; + } else if !omitted { + lines.push(line); + } + } + + lines.join("\n") + } + + let xform = |p: &Path| { + let mut contents = t!(fs::read_to_string(p)); + for tool in &["rust-demangler", "miri", "rust-docs"] { + if !built_tools.contains(tool) { + contents = filter(&contents, tool); + } + } + let ret = tmp.join(p.file_name().unwrap()); + t!(fs::write(&ret, &contents)); + ret + }; + + if target.contains("apple-darwin") { + builder.info("building pkg installer"); + let pkg = tmp.join("pkg"); + let _ = fs::remove_dir_all(&pkg); + + let pkgbuild = |component: &str| { + let mut cmd = Command::new("pkgbuild"); + cmd.arg("--identifier") + .arg(format!("org.rust-lang.{}", component)) + .arg("--scripts") + .arg(pkg.join(component)) + .arg("--nopayload") + .arg(pkg.join(component).with_extension("pkg")); + builder.run(&mut cmd); + }; + + let prepare = |name: &str| { + builder.create_dir(&pkg.join(name)); + builder.cp_r( + &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)), + &pkg.join(name), + ); + builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755); + pkgbuild(name); + }; + prepare("rustc"); + prepare("cargo"); + prepare("rust-std"); + prepare("rust-analysis"); + prepare("clippy"); + prepare("rust-analyzer"); + for tool in &["rust-docs", "rust-demangler", "miri", "rustc-codegen-cranelift"] { + if built_tools.contains(tool) { + prepare(tool); + } + } + // create an 'uninstall' package + builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755); + pkgbuild("uninstall"); + + builder.create_dir(&pkg.join("res")); + builder.create(&pkg.join("res/LICENSE.txt"), &license); + builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644); + let mut cmd = Command::new("productbuild"); + cmd.arg("--distribution") + .arg(xform(&etc.join("pkg/Distribution.xml"))) + .arg("--resources") + .arg(pkg.join("res")) + .arg(distdir(builder).join(format!( + "{}-{}.pkg", + pkgname(builder, "rust"), + target.triple + ))) + .arg("--package-path") + .arg(&pkg); + let _time = timeit(builder); + builder.run(&mut cmd); + } + + if target.contains("windows") { + let exe = tmp.join("exe"); + let _ = fs::remove_dir_all(&exe); + + let prepare = |name: &str| { + builder.create_dir(&exe.join(name)); + let dir = if name == "rust-std" || name == "rust-analysis" { + format!("{}-{}", name, target.triple) + } else if name == "rust-analyzer" { + "rust-analyzer-preview".to_string() + } else if name == "clippy" { + "clippy-preview".to_string() + } else if name == "rust-demangler" { + "rust-demangler-preview".to_string() + } else if name == "miri" { + "miri-preview".to_string() + } else if name == "rustc-codegen-cranelift" { + // FIXME add installer support for cg_clif once it is ready to be distributed on + // windows. + unreachable!("cg_clif shouldn't be built for windows"); + } else { + name.to_string() + }; + builder.cp_r( + &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir), + &exe.join(name), + ); + builder.remove(&exe.join(name).join("manifest.in")); + }; + prepare("rustc"); + prepare("cargo"); + prepare("rust-analysis"); + prepare("rust-std"); + for tool in &["clippy", "rust-analyzer", "rust-docs", "rust-demangler", "miri"] { + if built_tools.contains(tool) { + prepare(tool); + } + } + if target.ends_with("windows-gnu") { + prepare("rust-mingw"); + } + + builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644); + + // Generate msi installer + let wix_path = env::var_os("WIX") + .expect("`WIX` environment variable must be set for generating MSI installer(s)."); + let wix = PathBuf::from(wix_path); + let heat = wix.join("bin/heat.exe"); + let candle = wix.join("bin/candle.exe"); + let light = wix.join("bin/light.exe"); + + let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"]; + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rustc") + .args(&heat_flags) + .arg("-cg") + .arg("RustcGroup") + .arg("-dr") + .arg("Rustc") + .arg("-var") + .arg("var.RustcDir") + .arg("-out") + .arg(exe.join("RustcGroup.wxs")), + ); + if built_tools.contains("rust-docs") { + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rust-docs") + .args(&heat_flags) + .arg("-cg") + .arg("DocsGroup") + .arg("-dr") + .arg("Docs") + .arg("-var") + .arg("var.DocsDir") + .arg("-out") + .arg(exe.join("DocsGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/squash-components.xsl")), + ); + } + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("cargo") + .args(&heat_flags) + .arg("-cg") + .arg("CargoGroup") + .arg("-dr") + .arg("Cargo") + .arg("-var") + .arg("var.CargoDir") + .arg("-out") + .arg(exe.join("CargoGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/remove-duplicates.xsl")), + ); + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rust-std") + .args(&heat_flags) + .arg("-cg") + .arg("StdGroup") + .arg("-dr") + .arg("Std") + .arg("-var") + .arg("var.StdDir") + .arg("-out") + .arg(exe.join("StdGroup.wxs")), + ); + if built_tools.contains("rust-analyzer") { + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rust-analyzer") + .args(&heat_flags) + .arg("-cg") + .arg("RustAnalyzerGroup") + .arg("-dr") + .arg("RustAnalyzer") + .arg("-var") + .arg("var.RustAnalyzerDir") + .arg("-out") + .arg(exe.join("RustAnalyzerGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/remove-duplicates.xsl")), + ); + } + if built_tools.contains("clippy") { + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("clippy") + .args(&heat_flags) + .arg("-cg") + .arg("ClippyGroup") + .arg("-dr") + .arg("Clippy") + .arg("-var") + .arg("var.ClippyDir") + .arg("-out") + .arg(exe.join("ClippyGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/remove-duplicates.xsl")), + ); + } + if built_tools.contains("rust-demangler") { + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rust-demangler") + .args(&heat_flags) + .arg("-cg") + .arg("RustDemanglerGroup") + .arg("-dr") + .arg("RustDemangler") + .arg("-var") + .arg("var.RustDemanglerDir") + .arg("-out") + .arg(exe.join("RustDemanglerGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/remove-duplicates.xsl")), + ); + } + if built_tools.contains("miri") { + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("miri") + .args(&heat_flags) + .arg("-cg") + .arg("MiriGroup") + .arg("-dr") + .arg("Miri") + .arg("-var") + .arg("var.MiriDir") + .arg("-out") + .arg(exe.join("MiriGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/remove-duplicates.xsl")), + ); + } + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rust-analysis") + .args(&heat_flags) + .arg("-cg") + .arg("AnalysisGroup") + .arg("-dr") + .arg("Analysis") + .arg("-var") + .arg("var.AnalysisDir") + .arg("-out") + .arg(exe.join("AnalysisGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/remove-duplicates.xsl")), + ); + if target.ends_with("windows-gnu") { + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rust-mingw") + .args(&heat_flags) + .arg("-cg") + .arg("GccGroup") + .arg("-dr") + .arg("Gcc") + .arg("-var") + .arg("var.GccDir") + .arg("-out") + .arg(exe.join("GccGroup.wxs")), + ); + } + + let candle = |input: &Path| { + let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj"); + let arch = if target.contains("x86_64") { "x64" } else { "x86" }; + let mut cmd = Command::new(&candle); + cmd.current_dir(&exe) + .arg("-nologo") + .arg("-dRustcDir=rustc") + .arg("-dCargoDir=cargo") + .arg("-dStdDir=rust-std") + .arg("-dAnalysisDir=rust-analysis") + .arg("-arch") + .arg(&arch) + .arg("-out") + .arg(&output) + .arg(&input); + add_env(builder, &mut cmd, target); + + if built_tools.contains("clippy") { + cmd.arg("-dClippyDir=clippy"); + } + if built_tools.contains("rust-docs") { + cmd.arg("-dDocsDir=rust-docs"); + } + if built_tools.contains("rust-demangler") { + cmd.arg("-dRustDemanglerDir=rust-demangler"); + } + if built_tools.contains("rust-analyzer") { + cmd.arg("-dRustAnalyzerDir=rust-analyzer"); + } + if built_tools.contains("miri") { + cmd.arg("-dMiriDir=miri"); + } + if target.ends_with("windows-gnu") { + cmd.arg("-dGccDir=rust-mingw"); + } + builder.run(&mut cmd); + }; + candle(&xform(&etc.join("msi/rust.wxs"))); + candle(&etc.join("msi/ui.wxs")); + candle(&etc.join("msi/rustwelcomedlg.wxs")); + candle("RustcGroup.wxs".as_ref()); + if built_tools.contains("rust-docs") { + candle("DocsGroup.wxs".as_ref()); + } + candle("CargoGroup.wxs".as_ref()); + candle("StdGroup.wxs".as_ref()); + if built_tools.contains("clippy") { + candle("ClippyGroup.wxs".as_ref()); + } + if built_tools.contains("miri") { + candle("MiriGroup.wxs".as_ref()); + } + if built_tools.contains("rust-demangler") { + candle("RustDemanglerGroup.wxs".as_ref()); + } + if built_tools.contains("rust-analyzer") { + candle("RustAnalyzerGroup.wxs".as_ref()); + } + candle("AnalysisGroup.wxs".as_ref()); + + if target.ends_with("windows-gnu") { + candle("GccGroup.wxs".as_ref()); + } + + builder.create(&exe.join("LICENSE.rtf"), &rtf); + builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644); + builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644); + + builder.info(&format!("building `msi` installer with {light:?}")); + let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple); + let mut cmd = Command::new(&light); + cmd.arg("-nologo") + .arg("-ext") + .arg("WixUIExtension") + .arg("-ext") + .arg("WixUtilExtension") + .arg("-out") + .arg(exe.join(&filename)) + .arg("rust.wixobj") + .arg("ui.wixobj") + .arg("rustwelcomedlg.wixobj") + .arg("RustcGroup.wixobj") + .arg("CargoGroup.wixobj") + .arg("StdGroup.wixobj") + .arg("AnalysisGroup.wixobj") + .current_dir(&exe); + + if built_tools.contains("clippy") { + cmd.arg("ClippyGroup.wixobj"); + } + if built_tools.contains("miri") { + cmd.arg("MiriGroup.wixobj"); + } + if built_tools.contains("rust-analyzer") { + cmd.arg("RustAnalyzerGroup.wixobj"); + } + if built_tools.contains("rust-demangler") { + cmd.arg("RustDemanglerGroup.wixobj"); + } + if built_tools.contains("rust-docs") { + cmd.arg("DocsGroup.wixobj"); + } + + if target.ends_with("windows-gnu") { + cmd.arg("GccGroup.wixobj"); + } + // ICE57 wrongly complains about the shortcuts + cmd.arg("-sice:ICE57"); + + let _time = timeit(builder); + builder.run(&mut cmd); + + if !builder.config.dry_run() { + t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename))); + } + } + } +} + +fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) { + let mut parts = builder.version.split('.'); + cmd.env("CFG_RELEASE_INFO", builder.rust_version()) + .env("CFG_RELEASE_NUM", &builder.version) + .env("CFG_RELEASE", builder.rust_release()) + .env("CFG_VER_MAJOR", parts.next().unwrap()) + .env("CFG_VER_MINOR", parts.next().unwrap()) + .env("CFG_VER_PATCH", parts.next().unwrap()) + .env("CFG_VER_BUILD", "0") // just needed to build + .env("CFG_PACKAGE_VERS", builder.rust_package_vers()) + .env("CFG_PACKAGE_NAME", pkgname(builder, "rust")) + .env("CFG_BUILD", target.triple) + .env("CFG_CHANNEL", &builder.config.channel); + + if target.contains("windows-gnullvm") { + cmd.env("CFG_MINGW", "1").env("CFG_ABI", "LLVM"); + } else if target.contains("windows-gnu") { + cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU"); + } else { + cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC"); + } +} + +fn install_llvm_file(builder: &Builder<'_>, source: &Path, destination: &Path) { + if builder.config.dry_run() { + return; + } + + builder.install(&source, destination, 0o644); +} + +/// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking. +/// +/// Returns whether the files were actually copied. +fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool { + if let Some(config) = builder.config.target_config.get(&target) { + if config.llvm_config.is_some() && !builder.config.llvm_from_ci { + // If the LLVM was externally provided, then we don't currently copy + // artifacts into the sysroot. This is not necessarily the right + // choice (in particular, it will require the LLVM dylib to be in + // the linker's load path at runtime), but the common use case for + // external LLVMs is distribution provided LLVMs, and in that case + // they're usually in the standard search path (e.g., /usr/lib) and + // copying them here is going to cause problems as we may end up + // with the wrong files and isn't what distributions want. + // + // This behavior may be revisited in the future though. + // + // If the LLVM is coming from ourselves (just from CI) though, we + // still want to install it, as it otherwise won't be available. + return false; + } + } + + // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib + // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely + // clear why this is the case, though. llvm-config will emit the versioned + // paths and we don't want those in the sysroot (as we're expecting + // unversioned paths). + if target.contains("apple-darwin") && builder.llvm_link_shared() { + let src_libdir = builder.llvm_out(target).join("lib"); + let llvm_dylib_path = src_libdir.join("libLLVM.dylib"); + if llvm_dylib_path.exists() { + builder.install(&llvm_dylib_path, dst_libdir, 0o644); + } + !builder.config.dry_run() + } else if let Ok(llvm::LlvmResult { llvm_config, .. }) = + llvm::prebuilt_llvm_config(builder, target) + { + let mut cmd = Command::new(llvm_config); + cmd.arg("--libfiles"); + builder.verbose(&format!("running {cmd:?}")); + let files = if builder.config.dry_run() { "".into() } else { output(&mut cmd) }; + let build_llvm_out = &builder.llvm_out(builder.config.build); + let target_llvm_out = &builder.llvm_out(target); + for file in files.trim_end().split(' ') { + // If we're not using a custom LLVM, make sure we package for the target. + let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) { + target_llvm_out.join(relative_path) + } else { + PathBuf::from(file) + }; + install_llvm_file(builder, &file, dst_libdir); + } + !builder.config.dry_run() + } else { + false + } +} + +/// Maybe add libLLVM.so to the target lib-dir for linking. +pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) { + let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib"); + // We do not need to copy LLVM files into the sysroot if it is not + // dynamically linked; it is already included into librustc_llvm + // statically. + if builder.llvm_link_shared() { + maybe_install_llvm(builder, target, &dst_libdir); + } +} + +/// Maybe add libLLVM.so to the runtime lib-dir for rustc itself. +pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) { + let dst_libdir = + sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target })); + // We do not need to copy LLVM files into the sysroot if it is not + // dynamically linked; it is already included into librustc_llvm + // statically. + if builder.llvm_link_shared() { + maybe_install_llvm(builder, target, &dst_libdir); + } +} + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct LlvmTools { + pub target: TargetSelection, +} + +impl Step for LlvmTools { + type Output = Option; + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(&run.builder, "llvm-tools"); + // FIXME: allow using the names of the tools themselves? + run.alias("llvm-tools").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(LlvmTools { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let target = self.target; + + /* run only if llvm-config isn't used */ + if let Some(config) = builder.config.target_config.get(&target) { + if let Some(ref _s) = config.llvm_config { + builder.info(&format!("Skipping LlvmTools ({target}): external LLVM")); + return None; + } + } + + builder.ensure(crate::core::build_steps::llvm::Llvm { target }); + + let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple); + tarball.set_overlay(OverlayKind::LLVM); + tarball.is_preview(true); + + // Prepare the image directory + let src_bindir = builder.llvm_out(target).join("bin"); + let dst_bindir = format!("lib/rustlib/{}/bin", target.triple); + for tool in LLVM_TOOLS { + let exe = src_bindir.join(exe(tool, target)); + tarball.add_file(&exe, &dst_bindir, 0o755); + } + + // Copy libLLVM.so to the target lib dir as well, so the RPATH like + // `$ORIGIN/../lib` can find it. It may also be used as a dependency + // of `rustc-dev` to support the inherited `-lLLVM` when using the + // compiler libraries. + maybe_install_llvm_target(builder, target, tarball.image_dir()); + + Some(tarball.generate()) + } +} + +// Tarball intended for internal consumption to ease rustc/std development. +// +// Should not be considered stable by end users. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct RustDev { + pub target: TargetSelection, +} + +impl Step for RustDev { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("rust-dev") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustDev { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let target = self.target; + + /* run only if llvm-config isn't used */ + if let Some(config) = builder.config.target_config.get(&target) { + if let Some(ref _s) = config.llvm_config { + builder.info(&format!("Skipping RustDev ({target}): external LLVM")); + return None; + } + } + + let mut tarball = Tarball::new(builder, "rust-dev", &target.triple); + tarball.set_overlay(OverlayKind::LLVM); + + builder.ensure(crate::core::build_steps::llvm::Llvm { target }); + + // We want to package `lld` to use it with `download-ci-llvm`. + builder.ensure(crate::core::build_steps::llvm::Lld { target }); + + let src_bindir = builder.llvm_out(target).join("bin"); + // If updating this, you likely want to change + // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users + // will not pick up the extra file until LLVM gets bumped. + // We should include all the build artifacts obtained from a source build, + // so that you can use the downloadable LLVM as if you’ve just run a full source build. + if src_bindir.exists() { + for entry in walkdir::WalkDir::new(&src_bindir) { + let entry = t!(entry); + if entry.file_type().is_file() && !entry.path_is_symlink() { + let name = entry.file_name().to_str().unwrap(); + tarball.add_file(src_bindir.join(name), "bin", 0o755); + } + } + } + + // We don't build LLD on some platforms, so only add it if it exists + let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target)); + if lld_path.exists() { + tarball.add_file(lld_path, "bin", 0o755); + } + + tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755); + + // Copy the include directory as well; needed mostly to build + // librustc_llvm properly (e.g., llvm-config.h is in here). But also + // just broadly useful to be able to link against the bundled LLVM. + tarball.add_dir(&builder.llvm_out(target).join("include"), "include"); + + // Copy libLLVM.so to the target lib dir as well, so the RPATH like + // `$ORIGIN/../lib` can find it. It may also be used as a dependency + // of `rustc-dev` to support the inherited `-lLLVM` when using the + // compiler libraries. + let dst_libdir = tarball.image_dir().join("lib"); + maybe_install_llvm(builder, target, &dst_libdir); + let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" }; + t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir); + + Some(tarball.generate()) + } +} + +// Tarball intended for internal consumption to ease rustc/std development. +// +// Should not be considered stable by end users. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct Bootstrap { + pub target: TargetSelection, +} + +impl Step for Bootstrap { + type Output = Option; + const DEFAULT: bool = false; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("bootstrap") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Bootstrap { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let target = self.target; + + let tarball = Tarball::new(builder, "bootstrap", &target.triple); + + let bootstrap_outdir = &builder.bootstrap_out; + for file in &["bootstrap", "rustc", "rustdoc", "sccache-plus-cl"] { + tarball.add_file(bootstrap_outdir.join(exe(file, target)), "bootstrap/bin", 0o755); + } + + Some(tarball.generate()) + } +} + +/// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the +/// release process to avoid cloning the monorepo and building stuff. +/// +/// Should not be considered stable by end users. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct BuildManifest { + pub target: TargetSelection, +} + +impl Step for BuildManifest { + type Output = GeneratedTarball; + const DEFAULT: bool = false; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("build-manifest") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(BuildManifest { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + let build_manifest = builder.tool_exe(Tool::BuildManifest); + + let tarball = Tarball::new(builder, "build-manifest", &self.target.triple); + tarball.add_file(&build_manifest, "bin", 0o755); + tarball.generate() + } +} + +/// Tarball containing artifacts necessary to reproduce the build of rustc. +/// +/// Currently this is the PGO profile data. +/// +/// Should not be considered stable by end users. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct ReproducibleArtifacts { + pub target: TargetSelection, +} + +impl Step for ReproducibleArtifacts { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("reproducible-artifacts") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(ReproducibleArtifacts { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + let mut added_anything = false; + let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple); + if let Some(path) = builder.config.rust_profile_use.as_ref() { + tarball.add_file(path, ".", 0o644); + added_anything = true; + } + if let Some(path) = builder.config.llvm_profile_use.as_ref() { + tarball.add_file(path, ".", 0o644); + added_anything = true; + } + for profile in &builder.config.reproducible_artifacts { + tarball.add_file(profile, ".", 0o644); + added_anything = true; + } + if added_anything { Some(tarball.generate()) } else { None } + } +} diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs new file mode 100644 index 000000000..e3fd942fc --- /dev/null +++ b/src/bootstrap/src/core/build_steps/doc.rs @@ -0,0 +1,1094 @@ +//! Documentation generation for rustbuilder. +//! +//! This module implements generation for all bits and pieces of documentation +//! for the Rust project. This notably includes suites like the rust book, the +//! nomicon, rust by example, standalone documentation, etc. +//! +//! Everything here is basically just a shim around calling either `rustbook` or +//! `rustdoc`. + +use std::fs; +use std::path::{Path, PathBuf}; + +use crate::core::build_steps::compile; +use crate::core::build_steps::tool::{self, prepare_tool_cargo, SourceType, Tool}; +use crate::core::builder::crate_description; +use crate::core::builder::{Alias, Builder, Compiler, Kind, RunConfig, ShouldRun, Step}; +use crate::core::config::{Config, TargetSelection}; +use crate::utils::cache::{Interned, INTERNER}; +use crate::utils::helpers::{dir_is_empty, symlink_dir, t, up_to_date}; +use crate::Mode; + +macro_rules! submodule_helper { + ($path:expr, submodule) => { + $path + }; + ($path:expr, submodule = $submodule:literal) => { + $submodule + }; +} + +macro_rules! book { + ($($name:ident, $path:expr, $book_name:expr $(, submodule $(= $submodule:literal)? )? ;)+) => { + $( + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] + pub struct $name { + target: TargetSelection, + } + + impl Step for $name { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path($path).default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure($name { + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) { + $( + let path = Path::new(submodule_helper!( $path, submodule $( = $submodule )? )); + builder.update_submodule(&path); + )? + builder.ensure(RustbookSrc { + target: self.target, + name: INTERNER.intern_str($book_name), + src: INTERNER.intern_path(builder.src.join($path)), + parent: Some(self), + }) + } + } + )+ + } +} + +// NOTE: When adding a book here, make sure to ALSO build the book by +// adding a build step in `src/bootstrap/builder.rs`! +// NOTE: Make sure to add the corresponding submodule when adding a new book. +// FIXME: Make checking for a submodule automatic somehow (maybe by having a list of all submodules +// and checking against it?). +book!( + CargoBook, "src/tools/cargo/src/doc", "cargo", submodule = "src/tools/cargo"; + ClippyBook, "src/tools/clippy/book", "clippy"; + EditionGuide, "src/doc/edition-guide", "edition-guide", submodule; + EmbeddedBook, "src/doc/embedded-book", "embedded-book", submodule; + Nomicon, "src/doc/nomicon", "nomicon", submodule; + Reference, "src/doc/reference", "reference", submodule; + RustByExample, "src/doc/rust-by-example", "rust-by-example", submodule; + RustdocBook, "src/doc/rustdoc", "rustdoc"; + StyleGuide, "src/doc/style-guide", "style-guide"; +); + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct UnstableBook { + target: TargetSelection, +} + +impl Step for UnstableBook { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/doc/unstable-book").default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(UnstableBook { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + builder.ensure(UnstableBookGen { target: self.target }); + builder.ensure(RustbookSrc { + target: self.target, + name: INTERNER.intern_str("unstable-book"), + src: INTERNER.intern_path(builder.md_doc_out(self.target).join("unstable-book")), + parent: Some(self), + }) + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +struct RustbookSrc { + target: TargetSelection, + name: Interned, + src: Interned, + parent: Option

, +} + +impl Step for RustbookSrc

{ + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Invoke `rustbook` for `target` for the doc book `name` from the `src` path. + /// + /// This will not actually generate any documentation if the documentation has + /// already been generated. + fn run(self, builder: &Builder<'_>) { + let target = self.target; + let name = self.name; + let src = self.src; + let out = builder.doc_out(target); + t!(fs::create_dir_all(&out)); + + let out = out.join(name); + let index = out.join("index.html"); + let rustbook = builder.tool_exe(Tool::Rustbook); + let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook); + + if !builder.config.dry_run() && !(up_to_date(&src, &index) || up_to_date(&rustbook, &index)) + { + builder.info(&format!("Rustbook ({target}) - {name}")); + let _ = fs::remove_dir_all(&out); + + builder.run(rustbook_cmd.arg("build").arg(&src).arg("-d").arg(out)); + } + + if self.parent.is_some() { + builder.maybe_open_in_browser::

(index) + } + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct TheBook { + compiler: Compiler, + target: TargetSelection, +} + +impl Step for TheBook { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/doc/book").default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(TheBook { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + /// Builds the book and associated stuff. + /// + /// We need to build: + /// + /// * Book + /// * Older edition redirects + /// * Version info and CSS + /// * Index page + /// * Redirect pages + fn run(self, builder: &Builder<'_>) { + let relative_path = Path::new("src").join("doc").join("book"); + builder.update_submodule(&relative_path); + + let compiler = self.compiler; + let target = self.target; + + let absolute_path = builder.src.join(&relative_path); + let redirect_path = absolute_path.join("redirects"); + if !absolute_path.exists() + || !redirect_path.exists() + || dir_is_empty(&absolute_path) + || dir_is_empty(&redirect_path) + { + eprintln!("Please checkout submodule: {}", relative_path.display()); + crate::exit!(1); + } + // build book + builder.ensure(RustbookSrc { + target, + name: INTERNER.intern_str("book"), + src: INTERNER.intern_path(absolute_path.clone()), + parent: Some(self), + }); + + // building older edition redirects + for edition in &["first-edition", "second-edition", "2018-edition"] { + builder.ensure(RustbookSrc { + target, + name: INTERNER.intern_string(format!("book/{edition}")), + src: INTERNER.intern_path(absolute_path.join(edition)), + // There should only be one book that is marked as the parent for each target, so + // treat the other editions as not having a parent. + parent: Option::::None, + }); + } + + // build the version info page and CSS + let shared_assets = builder.ensure(SharedAssets { target }); + + // build the command first so we don't nest GHA groups + builder.rustdoc_cmd(compiler); + + // build the redirect pages + let _guard = builder.msg_doc(compiler, "book redirect pages", target); + for file in t!(fs::read_dir(redirect_path)) { + let file = t!(file); + let path = file.path(); + let path = path.to_str().unwrap(); + + invoke_rustdoc(builder, compiler, &shared_assets, target, path); + } + } +} + +fn invoke_rustdoc( + builder: &Builder<'_>, + compiler: Compiler, + shared_assets: &SharedAssetsPaths, + target: TargetSelection, + markdown: &str, +) { + let out = builder.doc_out(target); + + let path = builder.src.join("src/doc").join(markdown); + + let header = builder.src.join("src/doc/redirect.inc"); + let footer = builder.src.join("src/doc/footer.inc"); + + let mut cmd = builder.rustdoc_cmd(compiler); + + let out = out.join("book"); + + cmd.arg("--html-after-content") + .arg(&footer) + .arg("--html-before-content") + .arg(&shared_assets.version_info) + .arg("--html-in-header") + .arg(&header) + .arg("--markdown-no-toc") + .arg("--markdown-playground-url") + .arg("https://play.rust-lang.org/") + .arg("-o") + .arg(&out) + .arg(&path) + .arg("--markdown-css") + .arg("../rust.css"); + + if !builder.config.docs_minification { + cmd.arg("-Z").arg("unstable-options").arg("--disable-minification"); + } + + builder.run(&mut cmd); +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Standalone { + compiler: Compiler, + target: TargetSelection, +} + +impl Step for Standalone { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/doc").alias("standalone").default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Standalone { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + /// Generates all standalone documentation as compiled by the rustdoc in `stage` + /// for the `target` into `out`. + /// + /// This will list all of `src/doc` looking for markdown files and appropriately + /// perform transformations like substituting `VERSION`, `SHORT_HASH`, and + /// `STAMP` along with providing the various header/footer HTML we've customized. + /// + /// In the end, this is just a glorified wrapper around rustdoc! + fn run(self, builder: &Builder<'_>) { + let target = self.target; + let compiler = self.compiler; + let _guard = builder.msg_doc(compiler, "standalone", target); + let out = builder.doc_out(target); + t!(fs::create_dir_all(&out)); + + let version_info = builder.ensure(SharedAssets { target: self.target }).version_info; + + let favicon = builder.src.join("src/doc/favicon.inc"); + let footer = builder.src.join("src/doc/footer.inc"); + let full_toc = builder.src.join("src/doc/full-toc.inc"); + + for file in t!(fs::read_dir(builder.src.join("src/doc"))) { + let file = t!(file); + let path = file.path(); + let filename = path.file_name().unwrap().to_str().unwrap(); + if !filename.ends_with(".md") || filename == "README.md" { + continue; + } + + let html = out.join(filename).with_extension("html"); + let rustdoc = builder.rustdoc(compiler); + if up_to_date(&path, &html) + && up_to_date(&footer, &html) + && up_to_date(&favicon, &html) + && up_to_date(&full_toc, &html) + && (builder.config.dry_run() || up_to_date(&version_info, &html)) + && (builder.config.dry_run() || up_to_date(&rustdoc, &html)) + { + continue; + } + + let mut cmd = builder.rustdoc_cmd(compiler); + // Needed for --index-page flag + cmd.arg("-Z").arg("unstable-options"); + + cmd.arg("--html-after-content") + .arg(&footer) + .arg("--html-before-content") + .arg(&version_info) + .arg("--html-in-header") + .arg(&favicon) + .arg("--markdown-no-toc") + .arg("--index-page") + .arg(&builder.src.join("src/doc/index.md")) + .arg("--markdown-playground-url") + .arg("https://play.rust-lang.org/") + .arg("-o") + .arg(&out) + .arg(&path); + + if !builder.config.docs_minification { + cmd.arg("--disable-minification"); + } + + if filename == "not_found.md" { + cmd.arg("--markdown-css").arg("https://doc.rust-lang.org/rust.css"); + } else { + cmd.arg("--markdown-css").arg("rust.css"); + } + builder.run(&mut cmd); + } + + // We open doc/index.html as the default if invoked as `x.py doc --open` + // with no particular explicit doc requested (e.g. library/core). + if builder.paths.is_empty() || builder.was_invoked_explicitly::(Kind::Doc) { + let index = out.join("index.html"); + builder.open_in_browser(&index); + } + } +} + +#[derive(Debug, Clone)] +pub struct SharedAssetsPaths { + pub version_info: PathBuf, +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct SharedAssets { + target: TargetSelection, +} + +impl Step for SharedAssets { + type Output = SharedAssetsPaths; + const DEFAULT: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + // Other tasks depend on this, no need to execute it on its own + run.never() + } + + // Generate shared resources used by other pieces of documentation. + fn run(self, builder: &Builder<'_>) -> Self::Output { + let out = builder.doc_out(self.target); + + let version_input = builder.src.join("src").join("doc").join("version_info.html.template"); + let version_info = out.join("version_info.html"); + if !builder.config.dry_run() && !up_to_date(&version_input, &version_info) { + let info = t!(fs::read_to_string(&version_input)) + .replace("VERSION", &builder.rust_release()) + .replace("SHORT_HASH", builder.rust_info().sha_short().unwrap_or("")) + .replace("STAMP", builder.rust_info().sha().unwrap_or("")); + t!(fs::write(&version_info, &info)); + } + + builder.copy(&builder.src.join("src").join("doc").join("rust.css"), &out.join("rust.css")); + + SharedAssetsPaths { version_info } + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] +pub struct Std { + pub stage: u32, + pub target: TargetSelection, + pub format: DocumentationFormat, + crates: Interned>, +} + +impl Std { + pub(crate) fn new( + stage: u32, + target: TargetSelection, + builder: &Builder<'_>, + format: DocumentationFormat, + ) -> Self { + let crates = builder + .in_tree_crates("sysroot", Some(target)) + .into_iter() + .map(|krate| krate.name.to_string()) + .collect(); + Std { stage, target, format, crates: INTERNER.intern_list(crates) } + } +} + +impl Step for Std { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.crate_or_deps("sysroot").path("library").default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Std { + stage: run.builder.top_stage, + target: run.target, + format: if run.builder.config.cmd.json() { + DocumentationFormat::JSON + } else { + DocumentationFormat::HTML + }, + crates: run.make_run_crates(Alias::Library), + }); + } + + /// Compile all standard library documentation. + /// + /// This will generate all documentation for the standard library and its + /// dependencies. This is largely just a wrapper around `cargo doc`. + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let target = self.target; + let out = match self.format { + DocumentationFormat::HTML => builder.doc_out(target), + DocumentationFormat::JSON => builder.json_doc_out(target), + }; + + t!(fs::create_dir_all(&out)); + + if self.format == DocumentationFormat::HTML { + builder.ensure(SharedAssets { target: self.target }); + } + + let index_page = builder + .src + .join("src/doc/index.md") + .into_os_string() + .into_string() + .expect("non-utf8 paths are unsupported"); + let mut extra_args = match self.format { + DocumentationFormat::HTML => { + vec!["--markdown-css", "rust.css", "--markdown-no-toc", "--index-page", &index_page] + } + DocumentationFormat::JSON => vec!["--output-format", "json"], + }; + + if !builder.config.docs_minification { + extra_args.push("--disable-minification"); + } + + doc_std(builder, self.format, stage, target, &out, &extra_args, &self.crates); + + // Don't open if the format is json + if let DocumentationFormat::JSON = self.format { + return; + } + + if builder.paths.iter().any(|path| path.ends_with("library")) { + // For `x.py doc library --open`, open `std` by default. + let index = out.join("std").join("index.html"); + builder.open_in_browser(index); + } else { + for requested_crate in &*self.crates { + if STD_PUBLIC_CRATES.iter().any(|&k| k == requested_crate) { + let index = out.join(requested_crate).join("index.html"); + builder.open_in_browser(index); + break; + } + } + } + } +} + +/// Name of the crates that are visible to consumers of the standard library. +/// Documentation for internal crates is handled by the rustc step, so internal crates will show +/// up there. +/// +/// Order here is important! +/// Crates need to be processed starting from the leaves, otherwise rustdoc will not +/// create correct links between crates because rustdoc depends on the +/// existence of the output directories to know if it should be a local +/// or remote link. +const STD_PUBLIC_CRATES: [&str; 5] = ["core", "alloc", "std", "proc_macro", "test"]; + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] +pub enum DocumentationFormat { + HTML, + JSON, +} + +impl DocumentationFormat { + fn as_str(&self) -> &str { + match self { + DocumentationFormat::HTML => "HTML", + DocumentationFormat::JSON => "JSON", + } + } +} + +/// Build the documentation for public standard library crates. +fn doc_std( + builder: &Builder<'_>, + format: DocumentationFormat, + stage: u32, + target: TargetSelection, + out: &Path, + extra_args: &[&str], + requested_crates: &[String], +) { + if builder.no_std(target) == Some(true) { + panic!( + "building std documentation for no_std target {target} is not supported\n\ + Set `docs = false` in the config to disable documentation, or pass `--skip library`." + ); + } + + let compiler = builder.compiler(stage, builder.config.build); + + let target_doc_dir_name = if format == DocumentationFormat::JSON { "json-doc" } else { "doc" }; + let target_dir = + builder.stage_out(compiler, Mode::Std).join(target.triple).join(target_doc_dir_name); + + // This is directory where the compiler will place the output of the command. + // We will then copy the files from this directory into the final `out` directory, the specified + // as a function parameter. + let out_dir = target_dir.join(target.triple).join("doc"); + + let mut cargo = builder.cargo(compiler, Mode::Std, SourceType::InTree, target, "doc"); + compile::std_cargo(builder, target, compiler.stage, &mut cargo); + cargo + .arg("--no-deps") + .arg("--target-dir") + .arg(&*target_dir.to_string_lossy()) + .arg("-Zskip-rustdoc-fingerprint") + .rustdocflag("-Z") + .rustdocflag("unstable-options") + .rustdocflag("--resource-suffix") + .rustdocflag(&builder.version); + for arg in extra_args { + cargo.rustdocflag(arg); + } + + if builder.config.library_docs_private_items { + cargo.rustdocflag("--document-private-items").rustdocflag("--document-hidden-items"); + } + + for krate in requested_crates { + if krate == "sysroot" { + // The sysroot crate is an implementation detail, don't include it in public docs. + continue; + } + cargo.arg("-p").arg(krate); + } + + let description = + format!("library{} in {} format", crate_description(&requested_crates), format.as_str()); + let _guard = builder.msg_doc(compiler, &description, target); + + builder.run(&mut cargo.into()); + builder.cp_r(&out_dir, &out); +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Rustc { + pub stage: u32, + pub target: TargetSelection, + crates: Interned>, +} + +impl Rustc { + pub(crate) fn new(stage: u32, target: TargetSelection, builder: &Builder<'_>) -> Self { + let crates = builder + .in_tree_crates("rustc-main", Some(target)) + .into_iter() + .map(|krate| krate.name.to_string()) + .collect(); + Self { stage, target, crates: INTERNER.intern_list(crates) } + } +} + +impl Step for Rustc { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.crate_or_deps("rustc-main") + .path("compiler") + .default_condition(builder.config.compiler_docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Rustc { + stage: run.builder.top_stage, + target: run.target, + crates: run.make_run_crates(Alias::Compiler), + }); + } + + /// Generates compiler documentation. + /// + /// This will generate all documentation for compiler and dependencies. + /// Compiler documentation is distributed separately, so we make sure + /// we do not merge it with the other documentation from std, test and + /// proc_macros. This is largely just a wrapper around `cargo doc`. + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let target = self.target; + + // This is the intended out directory for compiler documentation. + let out = builder.compiler_doc_out(target); + t!(fs::create_dir_all(&out)); + + // Build the standard library, so that proc-macros can use it. + // (Normally, only the metadata would be necessary, but proc-macros are special since they run at compile-time.) + let compiler = builder.compiler(stage, builder.config.build); + builder.ensure(compile::Std::new(compiler, builder.config.build)); + + let _guard = builder.msg_sysroot_tool( + Kind::Doc, + stage, + &format!("compiler{}", crate_description(&self.crates)), + compiler.host, + target, + ); + + // Build cargo command. + let mut cargo = builder.cargo(compiler, Mode::Rustc, SourceType::InTree, target, "doc"); + cargo.rustdocflag("--document-private-items"); + // Since we always pass --document-private-items, there's no need to warn about linking to private items. + cargo.rustdocflag("-Arustdoc::private-intra-doc-links"); + cargo.rustdocflag("--enable-index-page"); + cargo.rustdocflag("-Zunstable-options"); + cargo.rustdocflag("-Znormalize-docs"); + cargo.rustdocflag("--show-type-layout"); + cargo.rustdocflag("--generate-link-to-definition"); + compile::rustc_cargo(builder, &mut cargo, target, compiler.stage); + cargo.arg("-Zunstable-options"); + cargo.arg("-Zskip-rustdoc-fingerprint"); + + // Only include compiler crates, no dependencies of those, such as `libc`. + // Do link to dependencies on `docs.rs` however using `rustdoc-map`. + cargo.arg("--no-deps"); + cargo.arg("-Zrustdoc-map"); + + // FIXME: `-Zrustdoc-map` does not yet correctly work for transitive dependencies, + // once this is no longer an issue the special case for `ena` can be removed. + cargo.rustdocflag("--extern-html-root-url"); + cargo.rustdocflag("ena=https://docs.rs/ena/latest/"); + + let mut to_open = None; + + let out_dir = builder.stage_out(compiler, Mode::Rustc).join(target.triple).join("doc"); + for krate in &*self.crates { + // Create all crate output directories first to make sure rustdoc uses + // relative links. + // FIXME: Cargo should probably do this itself. + let dir_name = krate.replace("-", "_"); + t!(fs::create_dir_all(out_dir.join(&*dir_name))); + cargo.arg("-p").arg(krate); + if to_open.is_none() { + to_open = Some(dir_name); + } + } + + // This uses a shared directory so that librustdoc documentation gets + // correctly built and merged with the rustc documentation. + // + // This is needed because rustdoc is built in a different directory from + // rustc. rustdoc needs to be able to see everything, for example when + // merging the search index, or generating local (relative) links. + symlink_dir_force(&builder.config, &out, &out_dir); + // Cargo puts proc macros in `target/doc` even if you pass `--target` + // explicitly (https://github.com/rust-lang/cargo/issues/7677). + let proc_macro_out_dir = builder.stage_out(compiler, Mode::Rustc).join("doc"); + symlink_dir_force(&builder.config, &out, &proc_macro_out_dir); + + builder.run(&mut cargo.into()); + + if !builder.config.dry_run() { + // Sanity check on linked compiler crates + for krate in &*self.crates { + let dir_name = krate.replace("-", "_"); + // Making sure the directory exists and is not empty. + assert!(out.join(&*dir_name).read_dir().unwrap().next().is_some()); + } + } + + if builder.paths.iter().any(|path| path.ends_with("compiler")) { + // For `x.py doc compiler --open`, open `rustc_middle` by default. + let index = out.join("rustc_middle").join("index.html"); + builder.open_in_browser(index); + } else if let Some(krate) = to_open { + // Let's open the first crate documentation page: + let index = out.join(krate).join("index.html"); + builder.open_in_browser(index); + } + } +} + +macro_rules! tool_doc { + ( + $tool: ident, + $should_run: literal, + $path: literal, + $(rustc_tool = $rustc_tool:literal, )? + $(in_tree = $in_tree:literal ,)? + $(is_library = $is_library:expr,)? + $(crates = $crates:expr)? + ) => { + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] + pub struct $tool { + target: TargetSelection, + } + + impl Step for $tool { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.crate_or_deps($should_run).default_condition(builder.config.compiler_docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure($tool { target: run.target }); + } + + /// Generates compiler documentation. + /// + /// This will generate all documentation for compiler and dependencies. + /// Compiler documentation is distributed separately, so we make sure + /// we do not merge it with the other documentation from std, test and + /// proc_macros. This is largely just a wrapper around `cargo doc`. + fn run(self, builder: &Builder<'_>) { + let stage = builder.top_stage; + let target = self.target; + + // This is the intended out directory for compiler documentation. + let out = builder.compiler_doc_out(target); + t!(fs::create_dir_all(&out)); + + let compiler = builder.compiler(stage, builder.config.build); + builder.ensure(compile::Std::new(compiler, target)); + + if true $(&& $rustc_tool)? { + // Build rustc docs so that we generate relative links. + builder.ensure(Rustc::new(stage, target, builder)); + + // Rustdoc needs the rustc sysroot available to build. + // FIXME: is there a way to only ensure `check::Rustc` here? Last time I tried it failed + // with strange errors, but only on a full bors test ... + builder.ensure(compile::Rustc::new(compiler, target)); + } + + let source_type = if true $(&& $in_tree)? { + SourceType::InTree + } else { + SourceType::Submodule + }; + + // Build cargo command. + let mut cargo = prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + target, + "doc", + $path, + source_type, + &[], + ); + + cargo.arg("-Zskip-rustdoc-fingerprint"); + // Only include compiler crates, no dependencies of those, such as `libc`. + cargo.arg("--no-deps"); + + if false $(|| $is_library)? { + cargo.arg("--lib"); + } + + $(for krate in $crates { + cargo.arg("-p").arg(krate); + })? + + cargo.rustdocflag("--document-private-items"); + // Since we always pass --document-private-items, there's no need to warn about linking to private items. + cargo.rustdocflag("-Arustdoc::private-intra-doc-links"); + cargo.rustdocflag("--enable-index-page"); + cargo.rustdocflag("--show-type-layout"); + cargo.rustdocflag("--generate-link-to-definition"); + cargo.rustdocflag("-Zunstable-options"); + + let out_dir = builder.stage_out(compiler, Mode::ToolRustc).join(target.triple).join("doc"); + $(for krate in $crates { + let dir_name = krate.replace("-", "_"); + t!(fs::create_dir_all(out_dir.join(&*dir_name))); + })? + + // Symlink compiler docs to the output directory of rustdoc documentation. + symlink_dir_force(&builder.config, &out, &out_dir); + let proc_macro_out_dir = builder.stage_out(compiler, Mode::ToolRustc).join("doc"); + symlink_dir_force(&builder.config, &out, &proc_macro_out_dir); + + let _guard = builder.msg_doc(compiler, stringify!($tool).to_lowercase(), target); + builder.run(&mut cargo.into()); + + if !builder.config.dry_run() { + // Sanity check on linked doc directories + $(for krate in $crates { + let dir_name = krate.replace("-", "_"); + // Making sure the directory exists and is not empty. + assert!(out.join(&*dir_name).read_dir().unwrap().next().is_some()); + })? + } + } + } + } +} + +tool_doc!(Rustdoc, "rustdoc-tool", "src/tools/rustdoc", crates = ["rustdoc", "rustdoc-json-types"]); +tool_doc!( + Rustfmt, + "rustfmt-nightly", + "src/tools/rustfmt", + crates = ["rustfmt-nightly", "rustfmt-config_proc_macro"] +); +tool_doc!(Clippy, "clippy", "src/tools/clippy", crates = ["clippy_config", "clippy_utils"]); +tool_doc!(Miri, "miri", "src/tools/miri", crates = ["miri"]); +tool_doc!( + Cargo, + "cargo", + "src/tools/cargo", + rustc_tool = false, + in_tree = false, + crates = [ + "cargo", + "cargo-platform", + "cargo-util", + "crates-io", + "cargo-test-macro", + "cargo-test-support", + "cargo-credential", + "mdman", + // FIXME: this trips a license check in tidy. + // "resolver-tests", + ] +); +tool_doc!(Tidy, "tidy", "src/tools/tidy", rustc_tool = false, crates = ["tidy"]); +tool_doc!( + Bootstrap, + "bootstrap", + "src/bootstrap", + rustc_tool = false, + is_library = true, + crates = ["bootstrap"] +); + +#[derive(Ord, PartialOrd, Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct ErrorIndex { + pub target: TargetSelection, +} + +impl Step for ErrorIndex { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/tools/error_index_generator").default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + let target = run.target; + run.builder.ensure(ErrorIndex { target }); + } + + /// Generates the HTML rendered error-index by running the + /// `error_index_generator` tool. + fn run(self, builder: &Builder<'_>) { + builder.info(&format!("Documenting error index ({})", self.target)); + let out = builder.doc_out(self.target); + t!(fs::create_dir_all(&out)); + let mut index = tool::ErrorIndex::command(builder); + index.arg("html"); + index.arg(out); + index.arg(&builder.version); + + builder.run(&mut index); + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct UnstableBookGen { + target: TargetSelection, +} + +impl Step for UnstableBookGen { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/tools/unstable-book-gen").default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(UnstableBookGen { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let target = self.target; + + builder.info(&format!("Generating unstable book md files ({target})")); + let out = builder.md_doc_out(target).join("unstable-book"); + builder.create_dir(&out); + builder.remove_dir(&out); + let mut cmd = builder.tool_cmd(Tool::UnstableBookGen); + cmd.arg(builder.src.join("library")); + cmd.arg(builder.src.join("compiler")); + cmd.arg(builder.src.join("src")); + cmd.arg(out); + + builder.run(&mut cmd); + } +} + +fn symlink_dir_force(config: &Config, original: &Path, link: &Path) { + if config.dry_run() { + return; + } + if let Ok(m) = fs::symlink_metadata(link) { + if m.file_type().is_dir() { + t!(fs::remove_dir_all(link)); + } else { + // handle directory junctions on windows by falling back to + // `remove_dir`. + t!(fs::remove_file(link).or_else(|_| fs::remove_dir(link))); + } + } + + t!( + symlink_dir(config, original, link), + format!("failed to create link from {} -> {}", link.display(), original.display()) + ); +} + +#[derive(Ord, PartialOrd, Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustcBook { + pub compiler: Compiler, + pub target: TargetSelection, + pub validate: bool, +} + +impl Step for RustcBook { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/doc/rustc").default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustcBook { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + validate: false, + }); + } + + /// Builds the rustc book. + /// + /// The lints are auto-generated by a tool, and then merged into the book + /// in the "md-doc" directory in the build output directory. Then + /// "rustbook" is used to convert it to HTML. + fn run(self, builder: &Builder<'_>) { + let out_base = builder.md_doc_out(self.target).join("rustc"); + t!(fs::create_dir_all(&out_base)); + let out_listing = out_base.join("src/lints"); + builder.cp_r(&builder.src.join("src/doc/rustc"), &out_base); + builder.info(&format!("Generating lint docs ({})", self.target)); + + let rustc = builder.rustc(self.compiler); + // The tool runs `rustc` for extracting output examples, so it needs a + // functional sysroot. + builder.ensure(compile::Std::new(self.compiler, self.target)); + let mut cmd = builder.tool_cmd(Tool::LintDocs); + cmd.arg("--src"); + cmd.arg(builder.src.join("compiler")); + cmd.arg("--out"); + cmd.arg(&out_listing); + cmd.arg("--rustc"); + cmd.arg(&rustc); + cmd.arg("--rustc-target").arg(&self.target.rustc_target_arg()); + if builder.is_verbose() { + cmd.arg("--verbose"); + } + if self.validate { + cmd.arg("--validate"); + } + // We need to validate nightly features, even on the stable channel. + // Set this unconditionally as the stage0 compiler may be being used to + // document. + cmd.env("RUSTC_BOOTSTRAP", "1"); + + // If the lib directories are in an unusual location (changed in + // config.toml), then this needs to explicitly update the dylib search + // path. + builder.add_rustc_lib_path(self.compiler, &mut cmd); + let doc_generator_guard = builder.msg( + Kind::Run, + self.compiler.stage, + "lint-docs", + self.compiler.host, + self.target, + ); + builder.run(&mut cmd); + drop(doc_generator_guard); + + // Run rustbook/mdbook to generate the HTML pages. + builder.ensure(RustbookSrc { + target: self.target, + name: INTERNER.intern_str("rustc"), + src: INTERNER.intern_path(out_base), + parent: Some(self), + }); + } +} diff --git a/src/bootstrap/src/core/build_steps/format.rs b/src/bootstrap/src/core/build_steps/format.rs new file mode 100644 index 000000000..86f1d925f --- /dev/null +++ b/src/bootstrap/src/core/build_steps/format.rs @@ -0,0 +1,322 @@ +//! Runs rustfmt on the repository. + +use crate::core::builder::Builder; +use crate::utils::helpers::{output, program_out_of_date, t}; +use build_helper::ci::CiEnv; +use build_helper::git::get_git_modified_files; +use ignore::WalkBuilder; +use std::collections::VecDeque; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::mpsc::SyncSender; + +fn rustfmt(src: &Path, rustfmt: &Path, paths: &[PathBuf], check: bool) -> impl FnMut(bool) -> bool { + let mut cmd = Command::new(&rustfmt); + // avoid the submodule config paths from coming into play, + // we only allow a single global config for the workspace for now + cmd.arg("--config-path").arg(&src.canonicalize().unwrap()); + cmd.arg("--edition").arg("2021"); + cmd.arg("--unstable-features"); + cmd.arg("--skip-children"); + if check { + cmd.arg("--check"); + } + cmd.args(paths); + let cmd_debug = format!("{cmd:?}"); + let mut cmd = cmd.spawn().expect("running rustfmt"); + // poor man's async: return a closure that'll wait for rustfmt's completion + move |block: bool| -> bool { + if !block { + match cmd.try_wait() { + Ok(Some(_)) => {} + _ => return false, + } + } + let status = cmd.wait().unwrap(); + if !status.success() { + eprintln!( + "Running `{}` failed.\nIf you're running `tidy`, \ + try again with `--bless`. Or, if you just want to format \ + code, run `./x.py fmt` instead.", + cmd_debug, + ); + crate::exit!(1); + } + true + } +} + +fn get_rustfmt_version(build: &Builder<'_>) -> Option<(String, PathBuf)> { + let stamp_file = build.out.join("rustfmt.stamp"); + + let mut cmd = Command::new(match build.initial_rustfmt() { + Some(p) => p, + None => return None, + }); + cmd.arg("--version"); + let output = match cmd.output() { + Ok(status) => status, + Err(_) => return None, + }; + if !output.status.success() { + return None; + } + Some((String::from_utf8(output.stdout).unwrap(), stamp_file)) +} + +/// Return whether the format cache can be reused. +fn verify_rustfmt_version(build: &Builder<'_>) -> bool { + let Some((version, stamp_file)) = get_rustfmt_version(build) else { + return false; + }; + !program_out_of_date(&stamp_file, &version) +} + +/// Updates the last rustfmt version used +fn update_rustfmt_version(build: &Builder<'_>) { + let Some((version, stamp_file)) = get_rustfmt_version(build) else { + return; + }; + t!(std::fs::write(stamp_file, version)) +} + +/// Returns the Rust files modified between the `merge-base` of HEAD and +/// rust-lang/master and what is now on the disk. +/// +/// Returns `None` if all files should be formatted. +fn get_modified_rs_files(build: &Builder<'_>) -> Result>, String> { + if !verify_rustfmt_version(build) { + return Ok(None); + } + + get_git_modified_files(&build.config.git_config(), Some(&build.config.src), &vec!["rs"]) +} + +#[derive(serde_derive::Deserialize)] +struct RustfmtConfig { + ignore: Vec, +} + +pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) { + if build.config.dry_run() { + return; + } + let mut builder = ignore::types::TypesBuilder::new(); + builder.add_defaults(); + builder.select("rust"); + let matcher = builder.build().unwrap(); + let rustfmt_config = build.src.join("rustfmt.toml"); + if !rustfmt_config.exists() { + eprintln!("Not running formatting checks; rustfmt.toml does not exist."); + eprintln!("This may happen in distributed tarballs."); + return; + } + let rustfmt_config = t!(std::fs::read_to_string(&rustfmt_config)); + let rustfmt_config: RustfmtConfig = t!(toml::from_str(&rustfmt_config)); + let mut fmt_override = ignore::overrides::OverrideBuilder::new(&build.src); + for ignore in rustfmt_config.ignore { + fmt_override.add(&format!("!{ignore}")).expect(&ignore); + } + let git_available = match Command::new("git") + .arg("--version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + { + Ok(status) => status.success(), + Err(_) => false, + }; + + if git_available { + let in_working_tree = match build + .config + .git() + .arg("rev-parse") + .arg("--is-inside-work-tree") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + { + Ok(status) => status.success(), + Err(_) => false, + }; + if in_working_tree { + let untracked_paths_output = output( + build.config.git().arg("status").arg("--porcelain").arg("--untracked-files=normal"), + ); + let untracked_paths = untracked_paths_output + .lines() + .filter(|entry| entry.starts_with("??")) + .map(|entry| { + entry.split(' ').nth(1).expect("every git status entry should list a path") + }); + let mut untracked_count = 0; + for untracked_path in untracked_paths { + println!("skip untracked path {untracked_path} during rustfmt invocations"); + // The leading `/` makes it an exact match against the + // repository root, rather than a glob. Without that, if you + // have `foo.rs` in the repository root it will also match + // against anything like `compiler/rustc_foo/src/foo.rs`, + // preventing the latter from being formatted. + untracked_count += 1; + fmt_override.add(&format!("!/{untracked_path}")).expect(&untracked_path); + } + // Only check modified files locally to speed up runtime. + // We still check all files in CI to avoid bugs in `get_modified_rs_files` letting regressions slip through; + // we also care about CI time less since this is still very fast compared to building the compiler. + if !CiEnv::is_ci() && paths.is_empty() { + match get_modified_rs_files(build) { + Ok(Some(files)) => { + if files.len() <= 10 { + for file in &files { + println!("formatting modified file {file}"); + } + } else { + let pluralized = |count| if count > 1 { "files" } else { "file" }; + let untracked_msg = if untracked_count == 0 { + "".to_string() + } else { + format!( + ", skipped {} untracked {}", + untracked_count, + pluralized(untracked_count), + ) + }; + println!( + "formatting {} modified {}{}", + files.len(), + pluralized(files.len()), + untracked_msg + ); + } + for file in files { + fmt_override.add(&format!("/{file}")).expect(&file); + } + } + Ok(None) => {} + Err(err) => { + println!( + "WARN: Something went wrong when running git commands:\n{err}\n\ + Falling back to formatting all files." + ); + } + } + } + } else { + println!("Not in git tree. Skipping git-aware format checks"); + } + } else { + println!("Could not find usable git. Skipping git-aware format checks"); + } + + let fmt_override = fmt_override.build().unwrap(); + + let rustfmt_path = build.initial_rustfmt().unwrap_or_else(|| { + eprintln!("./x.py fmt is not supported on this channel"); + crate::exit!(1); + }); + assert!(rustfmt_path.exists(), "{}", rustfmt_path.display()); + let src = build.src.clone(); + let (tx, rx): (SyncSender, _) = std::sync::mpsc::sync_channel(128); + let walker = match paths.get(0) { + Some(first) => { + let find_shortcut_candidates = |p: &PathBuf| { + let mut candidates = Vec::new(); + for candidate in WalkBuilder::new(src.clone()).max_depth(Some(3)).build() { + if let Ok(entry) = candidate { + if let Some(dir_name) = p.file_name() { + if entry.path().is_dir() && entry.file_name() == dir_name { + candidates.push(entry.into_path()); + } + } + } + } + candidates + }; + + // Only try to look for shortcut candidates for single component paths like + // `std` and not for e.g. relative paths like `../library/std`. + let should_look_for_shortcut_dir = |p: &PathBuf| p.components().count() == 1; + + let mut walker = if should_look_for_shortcut_dir(first) { + if let [single_candidate] = &find_shortcut_candidates(first)[..] { + WalkBuilder::new(single_candidate) + } else { + WalkBuilder::new(first) + } + } else { + WalkBuilder::new(src.join(first)) + }; + + for path in &paths[1..] { + if should_look_for_shortcut_dir(path) { + if let [single_candidate] = &find_shortcut_candidates(path)[..] { + walker.add(single_candidate); + } else { + walker.add(path); + } + } else { + walker.add(src.join(path)); + } + } + + walker + } + None => WalkBuilder::new(src.clone()), + } + .types(matcher) + .overrides(fmt_override) + .build_parallel(); + + // there is a lot of blocking involved in spawning a child process and reading files to format. + // spawn more processes than available concurrency to keep the CPU busy + let max_processes = build.jobs() as usize * 2; + + // spawn child processes on a separate thread so we can batch entries we have received from ignore + let thread = std::thread::spawn(move || { + let mut children = VecDeque::new(); + while let Ok(path) = rx.recv() { + // try getting a few more paths from the channel to amortize the overhead of spawning processes + let paths: Vec<_> = rx.try_iter().take(7).chain(std::iter::once(path)).collect(); + + let child = rustfmt(&src, &rustfmt_path, paths.as_slice(), check); + children.push_back(child); + + // poll completion before waiting + for i in (0..children.len()).rev() { + if children[i](false) { + children.swap_remove_back(i); + break; + } + } + + if children.len() >= max_processes { + // await oldest child + children.pop_front().unwrap()(true); + } + } + + // await remaining children + for mut child in children { + child(true); + } + }); + + walker.run(|| { + let tx = tx.clone(); + Box::new(move |entry| { + let entry = t!(entry); + if entry.file_type().map_or(false, |t| t.is_file()) { + t!(tx.send(entry.into_path())); + } + ignore::WalkState::Continue + }) + }); + + drop(tx); + + thread.join().unwrap(); + if !check { + update_rustfmt_version(build); + } +} diff --git a/src/bootstrap/src/core/build_steps/install.rs b/src/bootstrap/src/core/build_steps/install.rs new file mode 100644 index 000000000..6b4a8f597 --- /dev/null +++ b/src/bootstrap/src/core/build_steps/install.rs @@ -0,0 +1,332 @@ +//! Implementation of the install aspects of the compiler. +//! +//! This module is responsible for installing the standard library, +//! compiler, and documentation. + +use std::env; +use std::fs; +use std::path::{Component, Path, PathBuf}; +use std::process::Command; + +use crate::core::build_steps::dist; +use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::core::config::{Config, TargetSelection}; +use crate::utils::helpers::t; +use crate::utils::tarball::GeneratedTarball; +use crate::INTERNER; +use crate::{Compiler, Kind}; + +#[cfg(target_os = "illumos")] +const SHELL: &str = "bash"; +#[cfg(not(target_os = "illumos"))] +const SHELL: &str = "sh"; + +// We have to run a few shell scripts, which choke quite a bit on both `\` +// characters and on `C:\` paths, so normalize both of them away. +fn sanitize_sh(path: &Path) -> String { + let path = path.to_str().unwrap().replace("\\", "/"); + return change_drive(unc_to_lfs(&path)).unwrap_or(path); + + fn unc_to_lfs(s: &str) -> &str { + s.strip_prefix("//?/").unwrap_or(s) + } + + fn change_drive(s: &str) -> Option { + let mut ch = s.chars(); + let drive = ch.next().unwrap_or('C'); + if ch.next() != Some(':') { + return None; + } + if ch.next() != Some('/') { + return None; + } + Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..])) + } +} + +fn is_dir_writable_for_user(dir: &PathBuf) -> bool { + let tmp = dir.join(".tmp"); + match fs::create_dir_all(&tmp) { + Ok(_) => { + fs::remove_dir_all(tmp).unwrap(); + true + } + Err(e) => { + if e.kind() == std::io::ErrorKind::PermissionDenied { + false + } else { + panic!("Failed the write access check for the current user. {}", e); + } + } + } +} + +fn install_sh( + builder: &Builder<'_>, + package: &str, + stage: u32, + host: Option, + tarball: &GeneratedTarball, +) { + let _guard = builder.msg(Kind::Install, stage, package, host, host); + + let prefix = default_path(&builder.config.prefix, "/usr/local"); + let sysconfdir = prefix.join(default_path(&builder.config.sysconfdir, "/etc")); + let destdir_env = env::var_os("DESTDIR").map(PathBuf::from); + + // Sanity checks on the write access of user. + // + // When the `DESTDIR` environment variable is present, there is no point to + // check write access for `prefix` and `sysconfdir` individually, as they + // are combined with the path from the `DESTDIR` environment variable. In + // this case, we only need to check the `DESTDIR` path, disregarding the + // `prefix` and `sysconfdir` paths. + if let Some(destdir) = &destdir_env { + assert!(is_dir_writable_for_user(destdir), "User doesn't have write access on DESTDIR."); + } else { + assert!( + is_dir_writable_for_user(&prefix), + "User doesn't have write access on `install.prefix` path in the `config.toml`.", + ); + assert!( + is_dir_writable_for_user(&sysconfdir), + "User doesn't have write access on `install.sysconfdir` path in `config.toml`." + ); + } + + let datadir = prefix.join(default_path(&builder.config.datadir, "share")); + let docdir = prefix.join(default_path(&builder.config.docdir, "share/doc/rust")); + let mandir = prefix.join(default_path(&builder.config.mandir, "share/man")); + let libdir = prefix.join(default_path(&builder.config.libdir, "lib")); + let bindir = prefix.join(&builder.config.bindir); // Default in config.rs + + let empty_dir = builder.out.join("tmp/empty_dir"); + t!(fs::create_dir_all(&empty_dir)); + + let mut cmd = Command::new(SHELL); + cmd.current_dir(&empty_dir) + .arg(sanitize_sh(&tarball.decompressed_output().join("install.sh"))) + .arg(format!("--prefix={}", prepare_dir(&destdir_env, prefix))) + .arg(format!("--sysconfdir={}", prepare_dir(&destdir_env, sysconfdir))) + .arg(format!("--datadir={}", prepare_dir(&destdir_env, datadir))) + .arg(format!("--docdir={}", prepare_dir(&destdir_env, docdir))) + .arg(format!("--bindir={}", prepare_dir(&destdir_env, bindir))) + .arg(format!("--libdir={}", prepare_dir(&destdir_env, libdir))) + .arg(format!("--mandir={}", prepare_dir(&destdir_env, mandir))) + .arg("--disable-ldconfig"); + builder.run(&mut cmd); + t!(fs::remove_dir_all(&empty_dir)); +} + +fn default_path(config: &Option, default: &str) -> PathBuf { + config.as_ref().cloned().unwrap_or_else(|| PathBuf::from(default)) +} + +fn prepare_dir(destdir_env: &Option, mut path: PathBuf) -> String { + // The DESTDIR environment variable is a standard way to install software in a subdirectory + // while keeping the original directory structure, even if the prefix or other directories + // contain absolute paths. + // + // More information on the environment variable is available here: + // https://www.gnu.org/prep/standards/html_node/DESTDIR.html + if let Some(destdir) = destdir_env { + let without_destdir = path.clone(); + path = destdir.clone(); + // Custom .join() which ignores disk roots. + for part in without_destdir.components() { + if let Component::Normal(s) = part { + path.push(s) + } + } + } + + // The installation command is not executed from the current directory, but from a temporary + // directory. To prevent relative paths from breaking this converts relative paths to absolute + // paths. std::fs::canonicalize is not used as that requires the path to actually be present. + if path.is_relative() { + path = std::env::current_dir().expect("failed to get the current directory").join(path); + assert!(path.is_absolute(), "could not make the path relative"); + } + + sanitize_sh(&path) +} + +macro_rules! install { + (($sel:ident, $builder:ident, $_config:ident), + $($name:ident, + $condition_name: ident = $path_or_alias: literal, + $default_cond:expr, + only_hosts: $only_hosts:expr, + $run_item:block $(, $c:ident)*;)+) => { + $( + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] + pub struct $name { + pub compiler: Compiler, + pub target: TargetSelection, + } + + impl $name { + #[allow(dead_code)] + fn should_build(config: &Config) -> bool { + config.extended && config.tools.as_ref() + .map_or(true, |t| t.contains($path_or_alias)) + } + } + + impl Step for $name { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = $only_hosts; + $(const $c: bool = true;)* + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let $_config = &run.builder.config; + run.$condition_name($path_or_alias).default_condition($default_cond) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure($name { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + fn run($sel, $builder: &Builder<'_>) { + $run_item + } + })+ + } +} + +install!((self, builder, _config), + Docs, path = "src/doc", _config.docs, only_hosts: false, { + let tarball = builder.ensure(dist::Docs { host: self.target }).expect("missing docs"); + install_sh(builder, "docs", self.compiler.stage, Some(self.target), &tarball); + }; + Std, path = "library/std", true, only_hosts: false, { + for target in &builder.targets { + // `expect` should be safe, only None when host != build, but this + // only runs when host == build + let tarball = builder.ensure(dist::Std { + compiler: self.compiler, + target: *target + }).expect("missing std"); + install_sh(builder, "std", self.compiler.stage, Some(*target), &tarball); + } + }; + Cargo, alias = "cargo", Self::should_build(_config), only_hosts: true, { + let tarball = builder + .ensure(dist::Cargo { compiler: self.compiler, target: self.target }) + .expect("missing cargo"); + install_sh(builder, "cargo", self.compiler.stage, Some(self.target), &tarball); + }; + RustAnalyzer, alias = "rust-analyzer", Self::should_build(_config), only_hosts: true, { + if let Some(tarball) = + builder.ensure(dist::RustAnalyzer { compiler: self.compiler, target: self.target }) + { + install_sh(builder, "rust-analyzer", self.compiler.stage, Some(self.target), &tarball); + } else { + builder.info( + &format!("skipping Install rust-analyzer stage{} ({})", self.compiler.stage, self.target), + ); + } + }; + Clippy, alias = "clippy", Self::should_build(_config), only_hosts: true, { + let tarball = builder + .ensure(dist::Clippy { compiler: self.compiler, target: self.target }) + .expect("missing clippy"); + install_sh(builder, "clippy", self.compiler.stage, Some(self.target), &tarball); + }; + Miri, alias = "miri", Self::should_build(_config), only_hosts: true, { + if let Some(tarball) = builder.ensure(dist::Miri { compiler: self.compiler, target: self.target }) { + install_sh(builder, "miri", self.compiler.stage, Some(self.target), &tarball); + } else { + // Miri is only available on nightly + builder.info( + &format!("skipping Install miri stage{} ({})", self.compiler.stage, self.target), + ); + } + }; + LlvmTools, alias = "llvm-tools", Self::should_build(_config), only_hosts: true, { + if let Some(tarball) = builder.ensure(dist::LlvmTools { target: self.target }) { + install_sh(builder, "llvm-tools", self.compiler.stage, Some(self.target), &tarball); + } else { + builder.info( + &format!("skipping llvm-tools stage{} ({}): external LLVM", self.compiler.stage, self.target), + ); + } + }; + Rustfmt, alias = "rustfmt", Self::should_build(_config), only_hosts: true, { + if let Some(tarball) = builder.ensure(dist::Rustfmt { + compiler: self.compiler, + target: self.target + }) { + install_sh(builder, "rustfmt", self.compiler.stage, Some(self.target), &tarball); + } else { + builder.info( + &format!("skipping Install Rustfmt stage{} ({})", self.compiler.stage, self.target), + ); + } + }; + RustDemangler, alias = "rust-demangler", Self::should_build(_config), only_hosts: true, { + // NOTE: Even though `should_build` may return true for `extended` default tools, + // dist::RustDemangler may still return None, unless the target-dependent `profiler` config + // is also true, or the `tools` array explicitly includes "rust-demangler". + if let Some(tarball) = builder.ensure(dist::RustDemangler { + compiler: self.compiler, + target: self.target + }) { + install_sh(builder, "rust-demangler", self.compiler.stage, Some(self.target), &tarball); + } else { + builder.info( + &format!("skipping Install RustDemangler stage{} ({})", + self.compiler.stage, self.target), + ); + } + }; + Rustc, path = "compiler/rustc", true, only_hosts: true, { + let tarball = builder.ensure(dist::Rustc { + compiler: builder.compiler(builder.top_stage, self.target), + }); + install_sh(builder, "rustc", self.compiler.stage, Some(self.target), &tarball); + }; + RustcCodegenCranelift, alias = "rustc-codegen-cranelift", Self::should_build(_config), only_hosts: true, { + if let Some(tarball) = builder.ensure(dist::CodegenBackend { + compiler: self.compiler, + backend: INTERNER.intern_str("cranelift"), + }) { + install_sh(builder, "rustc-codegen-cranelift", self.compiler.stage, Some(self.target), &tarball); + } else { + builder.info( + &format!("skipping Install CodegenBackend(\"cranelift\") stage{} ({})", + self.compiler.stage, self.target), + ); + } + }; +); + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Src { + pub stage: u32, +} + +impl Step for Src { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let config = &run.builder.config; + let cond = config.extended && config.tools.as_ref().map_or(true, |t| t.contains("src")); + run.path("src").default_condition(cond) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Src { stage: run.builder.top_stage }); + } + + fn run(self, builder: &Builder<'_>) { + let tarball = builder.ensure(dist::Src); + install_sh(builder, "src", self.stage, None, &tarball); + } +} diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs new file mode 100644 index 000000000..a1f6fac8a --- /dev/null +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -0,0 +1,1366 @@ +//! Compilation of native dependencies like LLVM. +//! +//! Native projects like LLVM unfortunately aren't suited just yet for +//! compilation in build scripts that Cargo has. This is because the +//! compilation takes a *very* long time but also because we don't want to +//! compile LLVM 3 times as part of a normal bootstrap (we want it cached). +//! +//! LLVM and compiler-rt are essentially just wired up to everything else to +//! ensure that they're always in place if needed. + +use std::env; +use std::env::consts::EXE_EXTENSION; +use std::ffi::{OsStr, OsString}; +use std::fs::{self, File}; +use std::io; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::core::config::{Config, TargetSelection}; +use crate::utils::channel; +use crate::utils::helpers::{self, exe, get_clang_cl_resource_dir, output, t, up_to_date}; +use crate::{CLang, GitRepo, Kind}; + +use build_helper::ci::CiEnv; +use build_helper::git::get_git_merge_base; + +#[derive(Clone)] +pub struct LlvmResult { + /// Path to llvm-config binary. + /// NB: This is always the host llvm-config! + pub llvm_config: PathBuf, + /// Path to LLVM cmake directory for the target. + pub llvm_cmake_dir: PathBuf, +} + +pub struct Meta { + stamp: HashStamp, + res: LlvmResult, + out_dir: PathBuf, + root: String, +} + +// Linker flags to pass to LLVM's CMake invocation. +#[derive(Debug, Clone, Default)] +struct LdFlags { + // CMAKE_EXE_LINKER_FLAGS + exe: OsString, + // CMAKE_SHARED_LINKER_FLAGS + shared: OsString, + // CMAKE_MODULE_LINKER_FLAGS + module: OsString, +} + +impl LdFlags { + fn push_all(&mut self, s: impl AsRef) { + let s = s.as_ref(); + self.exe.push(" "); + self.exe.push(s); + self.shared.push(" "); + self.shared.push(s); + self.module.push(" "); + self.module.push(s); + } +} + +/// This returns whether we've already previously built LLVM. +/// +/// It's used to avoid busting caches during x.py check -- if we've already built +/// LLVM, it's fine for us to not try to avoid doing so. +/// +/// This will return the llvm-config if it can get it (but it will not build it +/// if not). +pub fn prebuilt_llvm_config( + builder: &Builder<'_>, + target: TargetSelection, +) -> Result { + builder.config.maybe_download_ci_llvm(); + + // If we're using a custom LLVM bail out here, but we can only use a + // custom LLVM for the build triple. + if let Some(config) = builder.config.target_config.get(&target) { + if let Some(ref s) = config.llvm_config { + check_llvm_version(builder, s); + let llvm_config = s.to_path_buf(); + let mut llvm_cmake_dir = llvm_config.clone(); + llvm_cmake_dir.pop(); + llvm_cmake_dir.pop(); + llvm_cmake_dir.push("lib"); + llvm_cmake_dir.push("cmake"); + llvm_cmake_dir.push("llvm"); + return Ok(LlvmResult { llvm_config, llvm_cmake_dir }); + } + } + + let root = "src/llvm-project/llvm"; + let out_dir = builder.llvm_out(target); + + let mut llvm_config_ret_dir = builder.llvm_out(builder.config.build); + if !builder.config.build.contains("msvc") || builder.ninja() { + llvm_config_ret_dir.push("build"); + } + llvm_config_ret_dir.push("bin"); + let build_llvm_config = llvm_config_ret_dir.join(exe("llvm-config", builder.config.build)); + let llvm_cmake_dir = out_dir.join("lib/cmake/llvm"); + let res = LlvmResult { llvm_config: build_llvm_config, llvm_cmake_dir }; + + let stamp = out_dir.join("llvm-finished-building"); + let stamp = HashStamp::new(stamp, builder.in_tree_llvm_info.sha()); + + if stamp.is_done() { + if stamp.hash.is_none() { + builder.info( + "Could not determine the LLVM submodule commit hash. \ + Assuming that an LLVM rebuild is not necessary.", + ); + builder.info(&format!( + "To force LLVM to rebuild, remove the file `{}`", + stamp.path.display() + )); + } + return Ok(res); + } + + Err(Meta { stamp, res, out_dir, root: root.into() }) +} + +/// This retrieves the LLVM sha we *want* to use, according to git history. +pub(crate) fn detect_llvm_sha(config: &Config, is_git: bool) -> String { + let llvm_sha = if is_git { + // We proceed in 2 steps. First we get the closest commit that is actually upstream. Then we + // walk back further to the last bors merge commit that actually changed LLVM. The first + // step will fail on CI because only the `auto` branch exists; we just fall back to `HEAD` + // in that case. + let closest_upstream = get_git_merge_base(&config.git_config(), Some(&config.src)) + .unwrap_or_else(|_| "HEAD".into()); + let mut rev_list = config.git(); + rev_list.args(&[ + PathBuf::from("rev-list"), + format!("--author={}", config.stage0_metadata.config.git_merge_commit_email).into(), + "-n1".into(), + "--first-parent".into(), + closest_upstream.into(), + "--".into(), + config.src.join("src/llvm-project"), + config.src.join("src/bootstrap/download-ci-llvm-stamp"), + // the LLVM shared object file is named `LLVM-12-rust-{version}-nightly` + config.src.join("src/version"), + ]); + output(&mut rev_list).trim().to_owned() + } else if let Some(info) = channel::read_commit_info_file(&config.src) { + info.sha.trim().to_owned() + } else { + "".to_owned() + }; + + if llvm_sha.is_empty() { + eprintln!("error: could not find commit hash for downloading LLVM"); + eprintln!("HELP: maybe your repository history is too shallow?"); + eprintln!("HELP: consider disabling `download-ci-llvm`"); + eprintln!("HELP: or fetch enough history to include one upstream commit"); + panic!(); + } + + llvm_sha +} + +/// Returns whether the CI-found LLVM is currently usable. +/// +/// This checks both the build triple platform to confirm we're usable at all, +/// and then verifies if the current HEAD matches the detected LLVM SHA head, +/// in which case LLVM is indicated as not available. +pub(crate) fn is_ci_llvm_available(config: &Config, asserts: bool) -> bool { + // This is currently all tier 1 targets and tier 2 targets with host tools + // (since others may not have CI artifacts) + // https://doc.rust-lang.org/rustc/platform-support.html#tier-1 + let supported_platforms = [ + // tier 1 + ("aarch64-unknown-linux-gnu", false), + ("i686-pc-windows-gnu", false), + ("i686-pc-windows-msvc", false), + ("i686-unknown-linux-gnu", false), + ("x86_64-unknown-linux-gnu", true), + ("x86_64-apple-darwin", true), + ("x86_64-pc-windows-gnu", true), + ("x86_64-pc-windows-msvc", true), + // tier 2 with host tools + ("aarch64-apple-darwin", false), + ("aarch64-pc-windows-msvc", false), + ("aarch64-unknown-linux-musl", false), + ("arm-unknown-linux-gnueabi", false), + ("arm-unknown-linux-gnueabihf", false), + ("armv7-unknown-linux-gnueabihf", false), + ("loongarch64-unknown-linux-gnu", false), + ("mips-unknown-linux-gnu", false), + ("mips64-unknown-linux-gnuabi64", false), + ("mips64el-unknown-linux-gnuabi64", false), + ("mipsel-unknown-linux-gnu", false), + ("powerpc-unknown-linux-gnu", false), + ("powerpc64-unknown-linux-gnu", false), + ("powerpc64le-unknown-linux-gnu", false), + ("riscv64gc-unknown-linux-gnu", false), + ("s390x-unknown-linux-gnu", false), + ("x86_64-unknown-freebsd", false), + ("x86_64-unknown-illumos", false), + ("x86_64-unknown-linux-musl", false), + ("x86_64-unknown-netbsd", false), + ]; + + if !supported_platforms.contains(&(&*config.build.triple, asserts)) + && (asserts || !supported_platforms.contains(&(&*config.build.triple, true))) + { + return false; + } + + if is_ci_llvm_modified(config) { + eprintln!("Detected LLVM as non-available: running in CI and modified LLVM in this change"); + return false; + } + + true +} + +/// Returns true if we're running in CI with modified LLVM (and thus can't download it) +pub(crate) fn is_ci_llvm_modified(config: &Config) -> bool { + CiEnv::is_ci() && config.rust_info.is_managed_git_subrepository() && { + // We assume we have access to git, so it's okay to unconditionally pass + // `true` here. + let llvm_sha = detect_llvm_sha(config, true); + let head_sha = output(config.git().arg("rev-parse").arg("HEAD")); + let head_sha = head_sha.trim(); + llvm_sha == head_sha + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Llvm { + pub target: TargetSelection, +} + +impl Step for Llvm { + type Output = LlvmResult; + + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/llvm-project").path("src/llvm-project/llvm") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Llvm { target: run.target }); + } + + /// Compile LLVM for `target`. + fn run(self, builder: &Builder<'_>) -> LlvmResult { + let target = self.target; + let target_native = if self.target.starts_with("riscv") { + // RISC-V target triples in Rust is not named the same as C compiler target triples. + // This converts Rust RISC-V target triples to C compiler triples. + let idx = target.triple.find('-').unwrap(); + + format!("riscv{}{}", &target.triple[5..7], &target.triple[idx..]) + } else if self.target.starts_with("powerpc") && self.target.ends_with("freebsd") { + // FreeBSD 13 had incompatible ABI changes on all PowerPC platforms. + // Set the version suffix to 13.0 so the correct target details are used. + format!("{}{}", self.target, "13.0") + } else { + target.to_string() + }; + + let Meta { stamp, res, out_dir, root } = match prebuilt_llvm_config(builder, target) { + Ok(p) => return p, + Err(m) => m, + }; + + builder.update_submodule(&Path::new("src").join("llvm-project")); + if builder.llvm_link_shared() && target.contains("windows") { + panic!("shared linking to LLVM is not currently supported on {}", target.triple); + } + + let _guard = builder.msg_unstaged(Kind::Build, "LLVM", target); + t!(stamp.remove()); + let _time = helpers::timeit(&builder); + t!(fs::create_dir_all(&out_dir)); + + // https://llvm.org/docs/CMake.html + let mut cfg = cmake::Config::new(builder.src.join(root)); + let mut ldflags = LdFlags::default(); + + let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) { + (false, _) => "Debug", + (true, false) => "Release", + (true, true) => "RelWithDebInfo", + }; + + // NOTE: remember to also update `config.example.toml` when changing the + // defaults! + let llvm_targets = match &builder.config.llvm_targets { + Some(s) => s, + None => { + "AArch64;ARM;BPF;Hexagon;LoongArch;MSP430;Mips;NVPTX;PowerPC;RISCV;\ + Sparc;SystemZ;WebAssembly;X86" + } + }; + + let llvm_exp_targets = match builder.config.llvm_experimental_targets { + Some(ref s) => s, + None => "AVR;M68k;CSKY", + }; + + let assertions = if builder.config.llvm_assertions { "ON" } else { "OFF" }; + let plugins = if builder.config.llvm_plugins { "ON" } else { "OFF" }; + let enable_tests = if builder.config.llvm_tests { "ON" } else { "OFF" }; + let enable_warnings = if builder.config.llvm_enable_warnings { "ON" } else { "OFF" }; + + cfg.out_dir(&out_dir) + .profile(profile) + .define("LLVM_ENABLE_ASSERTIONS", assertions) + .define("LLVM_UNREACHABLE_OPTIMIZE", "OFF") + .define("LLVM_ENABLE_PLUGINS", plugins) + .define("LLVM_TARGETS_TO_BUILD", llvm_targets) + .define("LLVM_EXPERIMENTAL_TARGETS_TO_BUILD", llvm_exp_targets) + .define("LLVM_INCLUDE_EXAMPLES", "OFF") + .define("LLVM_INCLUDE_DOCS", "OFF") + .define("LLVM_INCLUDE_BENCHMARKS", "OFF") + .define("LLVM_INCLUDE_TESTS", enable_tests) + .define("LLVM_ENABLE_TERMINFO", "OFF") + .define("LLVM_ENABLE_LIBEDIT", "OFF") + .define("LLVM_ENABLE_BINDINGS", "OFF") + .define("LLVM_ENABLE_Z3_SOLVER", "OFF") + .define("LLVM_PARALLEL_COMPILE_JOBS", builder.jobs().to_string()) + .define("LLVM_TARGET_ARCH", target_native.split('-').next().unwrap()) + .define("LLVM_DEFAULT_TARGET_TRIPLE", target_native) + .define("LLVM_ENABLE_WARNINGS", enable_warnings); + + // Parts of our test suite rely on the `FileCheck` tool, which is built by default in + // `build/$TARGET/llvm/build/bin` is but *not* then installed to `build/$TARGET/llvm/bin`. + // This flag makes sure `FileCheck` is copied in the final binaries directory. + cfg.define("LLVM_INSTALL_UTILS", "ON"); + + if builder.config.llvm_profile_generate { + cfg.define("LLVM_BUILD_INSTRUMENTED", "IR"); + if let Ok(llvm_profile_dir) = std::env::var("LLVM_PROFILE_DIR") { + cfg.define("LLVM_PROFILE_DATA_DIR", llvm_profile_dir); + } + cfg.define("LLVM_BUILD_RUNTIME", "No"); + } + if let Some(path) = builder.config.llvm_profile_use.as_ref() { + cfg.define("LLVM_PROFDATA_FILE", &path); + } + + // Disable zstd to avoid a dependency on libzstd.so. + cfg.define("LLVM_ENABLE_ZSTD", "OFF"); + + if !target.contains("windows") { + cfg.define("LLVM_ENABLE_ZLIB", "ON"); + } else { + cfg.define("LLVM_ENABLE_ZLIB", "OFF"); + } + + // Are we compiling for iOS/tvOS/watchOS? + if target.contains("apple-ios") + || target.contains("apple-tvos") + || target.contains("apple-watchos") + { + // These two defines prevent CMake from automatically trying to add a MacOSX sysroot, which leads to a compiler error. + cfg.define("CMAKE_OSX_SYSROOT", "/"); + cfg.define("CMAKE_OSX_DEPLOYMENT_TARGET", ""); + // Prevent cmake from adding -bundle to CFLAGS automatically, which leads to a compiler error because "-bitcode_bundle" also gets added. + cfg.define("LLVM_ENABLE_PLUGINS", "OFF"); + // Zlib fails to link properly, leading to a compiler error. + cfg.define("LLVM_ENABLE_ZLIB", "OFF"); + } + + // This setting makes the LLVM tools link to the dynamic LLVM library, + // which saves both memory during parallel links and overall disk space + // for the tools. We don't do this on every platform as it doesn't work + // equally well everywhere. + if builder.llvm_link_shared() { + cfg.define("LLVM_LINK_LLVM_DYLIB", "ON"); + } + + if (target.starts_with("riscv") || target.starts_with("csky")) + && !target.contains("freebsd") + && !target.contains("openbsd") + && !target.contains("netbsd") + { + // RISC-V and CSKY GCC erroneously requires linking against + // `libatomic` when using 1-byte and 2-byte C++ + // atomics but the LLVM build system check cannot + // detect this. Therefore it is set manually here. + // Some BSD uses Clang as its system compiler and + // provides no libatomic in its base system so does + // not want this. + ldflags.exe.push(" -latomic"); + ldflags.shared.push(" -latomic"); + } + + if target.starts_with("mips") && target.contains("netbsd") { + // LLVM wants 64-bit atomics, while mipsel is 32-bit only, so needs -latomic + ldflags.exe.push(" -latomic"); + ldflags.shared.push(" -latomic"); + } + + if target.contains("msvc") { + cfg.define("LLVM_USE_CRT_DEBUG", "MT"); + cfg.define("LLVM_USE_CRT_RELEASE", "MT"); + cfg.define("LLVM_USE_CRT_RELWITHDEBINFO", "MT"); + cfg.static_crt(true); + } + + if target.starts_with("i686") { + cfg.define("LLVM_BUILD_32_BITS", "ON"); + } + + let mut enabled_llvm_projects = Vec::new(); + + if helpers::forcing_clang_based_tests() { + enabled_llvm_projects.push("clang"); + enabled_llvm_projects.push("compiler-rt"); + } + + if builder.config.llvm_polly { + enabled_llvm_projects.push("polly"); + } + + if builder.config.llvm_clang { + enabled_llvm_projects.push("clang"); + } + + // We want libxml to be disabled. + // See https://github.com/rust-lang/rust/pull/50104 + cfg.define("LLVM_ENABLE_LIBXML2", "OFF"); + + if !enabled_llvm_projects.is_empty() { + enabled_llvm_projects.sort(); + enabled_llvm_projects.dedup(); + cfg.define("LLVM_ENABLE_PROJECTS", enabled_llvm_projects.join(";")); + } + + if let Some(num_linkers) = builder.config.llvm_link_jobs { + if num_linkers > 0 { + cfg.define("LLVM_PARALLEL_LINK_JOBS", num_linkers.to_string()); + } + } + + // https://llvm.org/docs/HowToCrossCompileLLVM.html + if target != builder.config.build { + let LlvmResult { llvm_config, .. } = + builder.ensure(Llvm { target: builder.config.build }); + if !builder.config.dry_run() { + let llvm_bindir = output(Command::new(&llvm_config).arg("--bindir")); + let host_bin = Path::new(llvm_bindir.trim()); + cfg.define( + "LLVM_TABLEGEN", + host_bin.join("llvm-tblgen").with_extension(EXE_EXTENSION), + ); + // LLVM_NM is required for cross compiling using MSVC + cfg.define("LLVM_NM", host_bin.join("llvm-nm").with_extension(EXE_EXTENSION)); + } + cfg.define("LLVM_CONFIG_PATH", llvm_config); + if builder.config.llvm_clang { + let build_bin = builder.llvm_out(builder.config.build).join("build").join("bin"); + let clang_tblgen = build_bin.join("clang-tblgen").with_extension(EXE_EXTENSION); + if !builder.config.dry_run() && !clang_tblgen.exists() { + panic!("unable to find {}", clang_tblgen.display()); + } + cfg.define("CLANG_TABLEGEN", clang_tblgen); + } + } + + let llvm_version_suffix = if let Some(ref suffix) = builder.config.llvm_version_suffix { + // Allow version-suffix="" to not define a version suffix at all. + if !suffix.is_empty() { Some(suffix.to_string()) } else { None } + } else if builder.config.channel == "dev" { + // Changes to a version suffix require a complete rebuild of the LLVM. + // To avoid rebuilds during a time of version bump, don't include rustc + // release number on the dev channel. + Some("-rust-dev".to_string()) + } else { + Some(format!("-rust-{}-{}", builder.version, builder.config.channel)) + }; + if let Some(ref suffix) = llvm_version_suffix { + cfg.define("LLVM_VERSION_SUFFIX", suffix); + } + + configure_cmake(builder, target, &mut cfg, true, ldflags, &[]); + configure_llvm(builder, target, &mut cfg); + + for (key, val) in &builder.config.llvm_build_config { + cfg.define(key, val); + } + + if builder.config.dry_run() { + return res; + } + + cfg.build(); + + // Helper to find the name of LLVM's shared library on darwin and linux. + let find_llvm_lib_name = |extension| { + let mut cmd = Command::new(&res.llvm_config); + let version = output(cmd.arg("--version")); + let major = version.split('.').next().unwrap(); + + match &llvm_version_suffix { + Some(version_suffix) => format!("libLLVM-{major}{version_suffix}.{extension}"), + None => format!("libLLVM-{major}.{extension}"), + } + }; + + // When building LLVM with LLVM_LINK_LLVM_DYLIB for macOS, an unversioned + // libLLVM.dylib will be built. However, llvm-config will still look + // for a versioned path like libLLVM-14.dylib. Manually create a symbolic + // link to make llvm-config happy. + if builder.llvm_link_shared() && target.contains("apple-darwin") { + let lib_name = find_llvm_lib_name("dylib"); + let lib_llvm = out_dir.join("build").join("lib").join(lib_name); + if !lib_llvm.exists() { + t!(builder.symlink_file("libLLVM.dylib", &lib_llvm)); + } + } + + // When building LLVM as a shared library on linux, it can contain unexpected debuginfo: + // some can come from the C++ standard library. Unless we're explicitly requesting LLVM to + // be built with debuginfo, strip it away after the fact, to make dist artifacts smaller. + if builder.llvm_link_shared() + && builder.config.llvm_optimize + && !builder.config.llvm_release_debuginfo + { + // Find the name of the LLVM shared library that we just built. + let lib_name = find_llvm_lib_name("so"); + + // If the shared library exists in LLVM's `/build/lib/` or `/lib/` folders, strip its + // debuginfo. + crate::core::build_steps::compile::strip_debug( + builder, + target, + &out_dir.join("lib").join(&lib_name), + ); + crate::core::build_steps::compile::strip_debug( + builder, + target, + &out_dir.join("build").join("lib").join(&lib_name), + ); + } + + t!(stamp.write()); + + res + } +} + +fn check_llvm_version(builder: &Builder<'_>, llvm_config: &Path) { + if builder.config.dry_run() { + return; + } + + let mut cmd = Command::new(llvm_config); + let version = output(cmd.arg("--version")); + let mut parts = version.split('.').take(2).filter_map(|s| s.parse::().ok()); + if let (Some(major), Some(_minor)) = (parts.next(), parts.next()) { + if major >= 15 { + return; + } + } + panic!("\n\nbad LLVM version: {version}, need >=15.0\n\n") +} + +fn configure_cmake( + builder: &Builder<'_>, + target: TargetSelection, + cfg: &mut cmake::Config, + use_compiler_launcher: bool, + mut ldflags: LdFlags, + extra_compiler_flags: &[&str], +) { + // Do not print installation messages for up-to-date files. + // LLVM and LLD builds can produce a lot of those and hit CI limits on log size. + cfg.define("CMAKE_INSTALL_MESSAGE", "LAZY"); + + // Do not allow the user's value of DESTDIR to influence where + // LLVM will install itself. LLVM must always be installed in our + // own build directories. + cfg.env("DESTDIR", ""); + + if builder.ninja() { + cfg.generator("Ninja"); + } + cfg.target(&target.triple).host(&builder.config.build.triple); + + if target != builder.config.build { + cfg.define("CMAKE_CROSSCOMPILING", "True"); + + if target.contains("netbsd") { + cfg.define("CMAKE_SYSTEM_NAME", "NetBSD"); + } else if target.contains("dragonfly") { + cfg.define("CMAKE_SYSTEM_NAME", "DragonFly"); + } else if target.contains("freebsd") { + cfg.define("CMAKE_SYSTEM_NAME", "FreeBSD"); + } else if target.contains("windows") { + cfg.define("CMAKE_SYSTEM_NAME", "Windows"); + } else if target.contains("haiku") { + cfg.define("CMAKE_SYSTEM_NAME", "Haiku"); + } else if target.contains("solaris") || target.contains("illumos") { + cfg.define("CMAKE_SYSTEM_NAME", "SunOS"); + } else if target.contains("linux") { + cfg.define("CMAKE_SYSTEM_NAME", "Linux"); + } else { + builder.info(&format!( + "could not determine CMAKE_SYSTEM_NAME from the target `{target}`, build may fail", + )); + } + + // When cross-compiling we should also set CMAKE_SYSTEM_VERSION, but in + // that case like CMake we cannot easily determine system version either. + // + // Since, the LLVM itself makes rather limited use of version checks in + // CMakeFiles (and then only in tests), and so far no issues have been + // reported, the system version is currently left unset. + + if target.contains("darwin") { + // Make sure that CMake does not build universal binaries on macOS. + // Explicitly specify the one single target architecture. + if target.starts_with("aarch64") { + // macOS uses a different name for building arm64 + cfg.define("CMAKE_OSX_ARCHITECTURES", "arm64"); + } else if target.starts_with("i686") { + // macOS uses a different name for building i386 + cfg.define("CMAKE_OSX_ARCHITECTURES", "i386"); + } else { + cfg.define("CMAKE_OSX_ARCHITECTURES", target.triple.split('-').next().unwrap()); + } + } + } + + let sanitize_cc = |cc: &Path| { + if target.contains("msvc") { + OsString::from(cc.to_str().unwrap().replace("\\", "/")) + } else { + cc.as_os_str().to_owned() + } + }; + + // MSVC with CMake uses msbuild by default which doesn't respect these + // vars that we'd otherwise configure. In that case we just skip this + // entirely. + if target.contains("msvc") && !builder.ninja() { + return; + } + + let (cc, cxx) = match builder.config.llvm_clang_cl { + Some(ref cl) => (cl.into(), cl.into()), + None => (builder.cc(target), builder.cxx(target).unwrap()), + }; + + // Handle msvc + ninja + ccache specially (this is what the bots use) + if target.contains("msvc") && builder.ninja() && builder.config.ccache.is_some() { + let mut wrap_cc = env::current_exe().expect("failed to get cwd"); + wrap_cc.set_file_name("sccache-plus-cl.exe"); + + cfg.define("CMAKE_C_COMPILER", sanitize_cc(&wrap_cc)) + .define("CMAKE_CXX_COMPILER", sanitize_cc(&wrap_cc)); + cfg.env("SCCACHE_PATH", builder.config.ccache.as_ref().unwrap()) + .env("SCCACHE_TARGET", target.triple) + .env("SCCACHE_CC", &cc) + .env("SCCACHE_CXX", &cxx); + + // Building LLVM on MSVC can be a little ludicrous at times. We're so far + // off the beaten path here that I'm not really sure this is even half + // supported any more. Here we're trying to: + // + // * Build LLVM on MSVC + // * Build LLVM with `clang-cl` instead of `cl.exe` + // * Build a project with `sccache` + // * Build for 32-bit as well + // * Build with Ninja + // + // For `cl.exe` there are different binaries to compile 32/64 bit which + // we use but for `clang-cl` there's only one which internally + // multiplexes via flags. As a result it appears that CMake's detection + // of a compiler's architecture and such on MSVC **doesn't** pass any + // custom flags we pass in CMAKE_CXX_FLAGS below. This means that if we + // use `clang-cl.exe` it's always diagnosed as a 64-bit compiler which + // definitely causes problems since all the env vars are pointing to + // 32-bit libraries. + // + // To hack around this... again... we pass an argument that's + // unconditionally passed in the sccache shim. This'll get CMake to + // correctly diagnose it's doing a 32-bit compilation and LLVM will + // internally configure itself appropriately. + if builder.config.llvm_clang_cl.is_some() && target.contains("i686") { + cfg.env("SCCACHE_EXTRA_ARGS", "-m32"); + } + } else { + // If ccache is configured we inform the build a little differently how + // to invoke ccache while also invoking our compilers. + if use_compiler_launcher { + if let Some(ref ccache) = builder.config.ccache { + cfg.define("CMAKE_C_COMPILER_LAUNCHER", ccache) + .define("CMAKE_CXX_COMPILER_LAUNCHER", ccache); + } + } + cfg.define("CMAKE_C_COMPILER", sanitize_cc(&cc)) + .define("CMAKE_CXX_COMPILER", sanitize_cc(&cxx)) + .define("CMAKE_ASM_COMPILER", sanitize_cc(&cc)); + } + + cfg.build_arg("-j").build_arg(builder.jobs().to_string()); + let mut cflags: OsString = builder.cflags(target, GitRepo::Llvm, CLang::C).join(" ").into(); + if let Some(ref s) = builder.config.llvm_cflags { + cflags.push(" "); + cflags.push(s); + } + + if builder.config.llvm_clang_cl.is_some() { + cflags.push(&format!(" --target={target}")); + } + for flag in extra_compiler_flags { + cflags.push(&format!(" {flag}")); + } + cfg.define("CMAKE_C_FLAGS", cflags); + let mut cxxflags: OsString = builder.cflags(target, GitRepo::Llvm, CLang::Cxx).join(" ").into(); + if let Some(ref s) = builder.config.llvm_cxxflags { + cxxflags.push(" "); + cxxflags.push(s); + } + if builder.config.llvm_clang_cl.is_some() { + cxxflags.push(&format!(" --target={target}")); + } + for flag in extra_compiler_flags { + cxxflags.push(&format!(" {flag}")); + } + cfg.define("CMAKE_CXX_FLAGS", cxxflags); + if let Some(ar) = builder.ar(target) { + if ar.is_absolute() { + // LLVM build breaks if `CMAKE_AR` is a relative path, for some reason it + // tries to resolve this path in the LLVM build directory. + cfg.define("CMAKE_AR", sanitize_cc(&ar)); + } + } + + if let Some(ranlib) = builder.ranlib(target) { + if ranlib.is_absolute() { + // LLVM build breaks if `CMAKE_RANLIB` is a relative path, for some reason it + // tries to resolve this path in the LLVM build directory. + cfg.define("CMAKE_RANLIB", sanitize_cc(&ranlib)); + } + } + + if let Some(ref flags) = builder.config.llvm_ldflags { + ldflags.push_all(flags); + } + + if let Some(flags) = get_var("LDFLAGS", &builder.config.build.triple, &target.triple) { + ldflags.push_all(&flags); + } + + // For distribution we want the LLVM tools to be *statically* linked to libstdc++. + // We also do this if the user explicitly requested static libstdc++. + if builder.config.llvm_static_stdcpp + && !target.contains("msvc") + && !target.contains("netbsd") + && !target.contains("solaris") + { + if target.contains("apple") || target.contains("windows") { + ldflags.push_all("-static-libstdc++"); + } else { + ldflags.push_all("-Wl,-Bsymbolic -static-libstdc++"); + } + } + + cfg.define("CMAKE_SHARED_LINKER_FLAGS", &ldflags.shared); + cfg.define("CMAKE_MODULE_LINKER_FLAGS", &ldflags.module); + cfg.define("CMAKE_EXE_LINKER_FLAGS", &ldflags.exe); + + if env::var_os("SCCACHE_ERROR_LOG").is_some() { + cfg.env("RUSTC_LOG", "sccache=warn"); + } +} + +fn configure_llvm(builder: &Builder<'_>, target: TargetSelection, cfg: &mut cmake::Config) { + // ThinLTO is only available when building with LLVM, enabling LLD is required. + // Apple's linker ld64 supports ThinLTO out of the box though, so don't use LLD on Darwin. + if builder.config.llvm_thin_lto { + cfg.define("LLVM_ENABLE_LTO", "Thin"); + if !target.contains("apple") { + cfg.define("LLVM_ENABLE_LLD", "ON"); + } + } + + if let Some(ref linker) = builder.config.llvm_use_linker { + cfg.define("LLVM_USE_LINKER", linker); + } + + if builder.config.llvm_allow_old_toolchain { + cfg.define("LLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN", "YES"); + } +} + +// Adapted from https://github.com/alexcrichton/cc-rs/blob/fba7feded71ee4f63cfe885673ead6d7b4f2f454/src/lib.rs#L2347-L2365 +fn get_var(var_base: &str, host: &str, target: &str) -> Option { + let kind = if host == target { "HOST" } else { "TARGET" }; + let target_u = target.replace("-", "_"); + env::var_os(&format!("{var_base}_{target}")) + .or_else(|| env::var_os(&format!("{}_{}", var_base, target_u))) + .or_else(|| env::var_os(&format!("{}_{}", kind, var_base))) + .or_else(|| env::var_os(var_base)) +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Lld { + pub target: TargetSelection, +} + +impl Step for Lld { + type Output = PathBuf; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/llvm-project/lld") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Lld { target: run.target }); + } + + /// Compile LLD for `target`. + fn run(self, builder: &Builder<'_>) -> PathBuf { + if builder.config.dry_run() { + return PathBuf::from("lld-out-dir-test-gen"); + } + let target = self.target; + + let LlvmResult { llvm_config, llvm_cmake_dir } = builder.ensure(Llvm { target }); + + // The `dist` step packages LLD next to LLVM's binaries for download-ci-llvm. The root path + // we usually expect here is `./build/$triple/ci-llvm/`, with the binaries in its `bin` + // subfolder. We check if that's the case, and if LLD's binary already exists there next to + // `llvm-config`: if so, we can use it instead of building LLVM/LLD from source. + let ci_llvm_bin = llvm_config.parent().unwrap(); + if ci_llvm_bin.is_dir() && ci_llvm_bin.file_name().unwrap() == "bin" { + let lld_path = ci_llvm_bin.join(exe("lld", target)); + if lld_path.exists() { + // The following steps copying `lld` as `rust-lld` to the sysroot, expect it in the + // `bin` subfolder of this step's out dir. + return ci_llvm_bin.parent().unwrap().to_path_buf(); + } + } + + let out_dir = builder.lld_out(target); + let done_stamp = out_dir.join("lld-finished-building"); + if done_stamp.exists() { + return out_dir; + } + + let _guard = builder.msg_unstaged(Kind::Build, "LLD", target); + let _time = helpers::timeit(&builder); + t!(fs::create_dir_all(&out_dir)); + + let mut cfg = cmake::Config::new(builder.src.join("src/llvm-project/lld")); + let mut ldflags = LdFlags::default(); + + // When building LLD as part of a build with instrumentation on windows, for example + // when doing PGO on CI, cmake or clang-cl don't automatically link clang's + // profiler runtime in. In that case, we need to manually ask cmake to do it, to avoid + // linking errors, much like LLVM's cmake setup does in that situation. + if builder.config.llvm_profile_generate && target.contains("msvc") { + if let Some(clang_cl_path) = builder.config.llvm_clang_cl.as_ref() { + // Find clang's runtime library directory and push that as a search path to the + // cmake linker flags. + let clang_rt_dir = get_clang_cl_resource_dir(clang_cl_path); + ldflags.push_all(&format!("/libpath:{}", clang_rt_dir.display())); + } + } + + // LLD is built as an LLVM tool, but is distributed outside of the `llvm-tools` component, + // which impacts where it expects to find LLVM's shared library. This causes #80703. + // + // LLD is distributed at "$root/lib/rustlib/$host/bin/rust-lld", but the `libLLVM-*.so` it + // needs is distributed at "$root/lib". The default rpath of "$ORIGIN/../lib" points at the + // lib path for LLVM tools, not the one for rust binaries. + // + // (The `llvm-tools` component copies the .so there for the other tools, and with that + // component installed, one can successfully invoke `rust-lld` directly without rustup's + // `LD_LIBRARY_PATH` overrides) + // + if builder.config.rpath_enabled(target) + && helpers::use_host_linker(target) + && builder.config.llvm_link_shared() + && target.contains("linux") + { + // So we inform LLD where it can find LLVM's libraries by adding an rpath entry to the + // expected parent `lib` directory. + // + // Be careful when changing this path, we need to ensure it's quoted or escaped: + // `$ORIGIN` would otherwise be expanded when the `LdFlags` are passed verbatim to + // cmake. + ldflags.push_all("-Wl,-rpath,'$ORIGIN/../../../'"); + } + + configure_cmake(builder, target, &mut cfg, true, ldflags, &[]); + configure_llvm(builder, target, &mut cfg); + + // Re-use the same flags as llvm to control the level of debug information + // generated for lld. + let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) { + (false, _) => "Debug", + (true, false) => "Release", + (true, true) => "RelWithDebInfo", + }; + + cfg.out_dir(&out_dir) + .profile(profile) + .define("LLVM_CMAKE_DIR", llvm_cmake_dir) + .define("LLVM_INCLUDE_TESTS", "OFF"); + + if target != builder.config.build { + // Use the host llvm-tblgen binary. + cfg.define( + "LLVM_TABLEGEN_EXE", + llvm_config.with_file_name("llvm-tblgen").with_extension(EXE_EXTENSION), + ); + } + + cfg.build(); + + t!(File::create(&done_stamp)); + out_dir + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Sanitizers { + pub target: TargetSelection, +} + +impl Step for Sanitizers { + type Output = Vec; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("sanitizers") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Sanitizers { target: run.target }); + } + + /// Builds sanitizer runtime libraries. + fn run(self, builder: &Builder<'_>) -> Self::Output { + let compiler_rt_dir = builder.src.join("src/llvm-project/compiler-rt"); + if !compiler_rt_dir.exists() { + return Vec::new(); + } + + let out_dir = builder.native_dir(self.target).join("sanitizers"); + let runtimes = supported_sanitizers(&out_dir, self.target, &builder.config.channel); + if runtimes.is_empty() { + return runtimes; + } + + let LlvmResult { llvm_config, .. } = builder.ensure(Llvm { target: builder.config.build }); + if builder.config.dry_run() { + return runtimes; + } + + let stamp = out_dir.join("sanitizers-finished-building"); + let stamp = HashStamp::new(stamp, builder.in_tree_llvm_info.sha()); + + if stamp.is_done() { + if stamp.hash.is_none() { + builder.info(&format!( + "Rebuild sanitizers by removing the file `{}`", + stamp.path.display() + )); + } + return runtimes; + } + + let _guard = builder.msg_unstaged(Kind::Build, "sanitizers", self.target); + t!(stamp.remove()); + let _time = helpers::timeit(&builder); + + let mut cfg = cmake::Config::new(&compiler_rt_dir); + cfg.profile("Release"); + cfg.define("CMAKE_C_COMPILER_TARGET", self.target.triple); + cfg.define("COMPILER_RT_BUILD_BUILTINS", "OFF"); + cfg.define("COMPILER_RT_BUILD_CRT", "OFF"); + cfg.define("COMPILER_RT_BUILD_LIBFUZZER", "OFF"); + cfg.define("COMPILER_RT_BUILD_PROFILE", "OFF"); + cfg.define("COMPILER_RT_BUILD_SANITIZERS", "ON"); + cfg.define("COMPILER_RT_BUILD_XRAY", "OFF"); + cfg.define("COMPILER_RT_DEFAULT_TARGET_ONLY", "ON"); + cfg.define("COMPILER_RT_USE_LIBCXX", "OFF"); + cfg.define("LLVM_CONFIG_PATH", &llvm_config); + + // On Darwin targets the sanitizer runtimes are build as universal binaries. + // Unfortunately sccache currently lacks support to build them successfully. + // Disable compiler launcher on Darwin targets to avoid potential issues. + let use_compiler_launcher = !self.target.contains("apple-darwin"); + let extra_compiler_flags: &[&str] = + if self.target.contains("apple") { &["-fembed-bitcode=off"] } else { &[] }; + configure_cmake( + builder, + self.target, + &mut cfg, + use_compiler_launcher, + LdFlags::default(), + extra_compiler_flags, + ); + + t!(fs::create_dir_all(&out_dir)); + cfg.out_dir(out_dir); + + for runtime in &runtimes { + cfg.build_target(&runtime.cmake_target); + cfg.build(); + } + t!(stamp.write()); + + runtimes + } +} + +#[derive(Clone, Debug)] +pub struct SanitizerRuntime { + /// CMake target used to build the runtime. + pub cmake_target: String, + /// Path to the built runtime library. + pub path: PathBuf, + /// Library filename that will be used rustc. + pub name: String, +} + +/// Returns sanitizers available on a given target. +fn supported_sanitizers( + out_dir: &Path, + target: TargetSelection, + channel: &str, +) -> Vec { + let darwin_libs = |os: &str, components: &[&str]| -> Vec { + components + .iter() + .map(move |c| SanitizerRuntime { + cmake_target: format!("clang_rt.{}_{}_dynamic", c, os), + path: out_dir + .join(&format!("build/lib/darwin/libclang_rt.{}_{}_dynamic.dylib", c, os)), + name: format!("librustc-{}_rt.{}.dylib", channel, c), + }) + .collect() + }; + + let common_libs = |os: &str, arch: &str, components: &[&str]| -> Vec { + components + .iter() + .map(move |c| SanitizerRuntime { + cmake_target: format!("clang_rt.{}-{}", c, arch), + path: out_dir.join(&format!("build/lib/{}/libclang_rt.{}-{}.a", os, c, arch)), + name: format!("librustc-{}_rt.{}.a", channel, c), + }) + .collect() + }; + + match &*target.triple { + "aarch64-apple-darwin" => darwin_libs("osx", &["asan", "lsan", "tsan"]), + "aarch64-apple-ios" => darwin_libs("ios", &["asan", "tsan"]), + "aarch64-apple-ios-sim" => darwin_libs("iossim", &["asan", "tsan"]), + "aarch64-apple-ios-macabi" => darwin_libs("osx", &["asan", "lsan", "tsan"]), + "aarch64-unknown-fuchsia" => common_libs("fuchsia", "aarch64", &["asan"]), + "aarch64-unknown-linux-gnu" => { + common_libs("linux", "aarch64", &["asan", "lsan", "msan", "tsan", "hwasan"]) + } + "aarch64-unknown-linux-ohos" => { + common_libs("linux", "aarch64", &["asan", "lsan", "msan", "tsan", "hwasan"]) + } + "x86_64-apple-darwin" => darwin_libs("osx", &["asan", "lsan", "tsan"]), + "x86_64-unknown-fuchsia" => common_libs("fuchsia", "x86_64", &["asan"]), + "x86_64-apple-ios" => darwin_libs("iossim", &["asan", "tsan"]), + "x86_64-apple-ios-macabi" => darwin_libs("osx", &["asan", "lsan", "tsan"]), + "x86_64-unknown-freebsd" => common_libs("freebsd", "x86_64", &["asan", "msan", "tsan"]), + "x86_64-unknown-netbsd" => { + common_libs("netbsd", "x86_64", &["asan", "lsan", "msan", "tsan"]) + } + "x86_64-unknown-illumos" => common_libs("illumos", "x86_64", &["asan"]), + "x86_64-pc-solaris" => common_libs("solaris", "x86_64", &["asan"]), + "x86_64-unknown-linux-gnu" => { + common_libs("linux", "x86_64", &["asan", "lsan", "msan", "safestack", "tsan"]) + } + "x86_64-unknown-linux-musl" => { + common_libs("linux", "x86_64", &["asan", "lsan", "msan", "tsan"]) + } + "s390x-unknown-linux-gnu" => { + common_libs("linux", "s390x", &["asan", "lsan", "msan", "tsan"]) + } + "s390x-unknown-linux-musl" => { + common_libs("linux", "s390x", &["asan", "lsan", "msan", "tsan"]) + } + "x86_64-unknown-linux-ohos" => { + common_libs("linux", "x86_64", &["asan", "lsan", "msan", "tsan"]) + } + _ => Vec::new(), + } +} + +struct HashStamp { + path: PathBuf, + hash: Option>, +} + +impl HashStamp { + fn new(path: PathBuf, hash: Option<&str>) -> Self { + HashStamp { path, hash: hash.map(|s| s.as_bytes().to_owned()) } + } + + fn is_done(&self) -> bool { + match fs::read(&self.path) { + Ok(h) => self.hash.as_deref().unwrap_or(b"") == h.as_slice(), + Err(e) if e.kind() == io::ErrorKind::NotFound => false, + Err(e) => { + panic!("failed to read stamp file `{}`: {}", self.path.display(), e); + } + } + } + + fn remove(&self) -> io::Result<()> { + match fs::remove_file(&self.path) { + Ok(()) => Ok(()), + Err(e) => { + if e.kind() == io::ErrorKind::NotFound { + Ok(()) + } else { + Err(e) + } + } + } + } + + fn write(&self) -> io::Result<()> { + fs::write(&self.path, self.hash.as_deref().unwrap_or(b"")) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct CrtBeginEnd { + pub target: TargetSelection, +} + +impl Step for CrtBeginEnd { + type Output = PathBuf; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/llvm-project/compiler-rt/lib/crt") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(CrtBeginEnd { target: run.target }); + } + + /// Build crtbegin.o/crtend.o for musl target. + fn run(self, builder: &Builder<'_>) -> Self::Output { + builder.update_submodule(&Path::new("src/llvm-project")); + + let out_dir = builder.native_dir(self.target).join("crt"); + + if builder.config.dry_run() { + return out_dir; + } + + let crtbegin_src = builder.src.join("src/llvm-project/compiler-rt/lib/builtins/crtbegin.c"); + let crtend_src = builder.src.join("src/llvm-project/compiler-rt/lib/builtins/crtend.c"); + if up_to_date(&crtbegin_src, &out_dir.join("crtbegin.o")) + && up_to_date(&crtend_src, &out_dir.join("crtendS.o")) + { + return out_dir; + } + + let _guard = builder.msg_unstaged(Kind::Build, "crtbegin.o and crtend.o", self.target); + t!(fs::create_dir_all(&out_dir)); + + let mut cfg = cc::Build::new(); + + if let Some(ar) = builder.ar(self.target) { + cfg.archiver(ar); + } + cfg.compiler(builder.cc(self.target)); + cfg.cargo_metadata(false) + .out_dir(&out_dir) + .target(&self.target.triple) + .host(&builder.config.build.triple) + .warnings(false) + .debug(false) + .opt_level(3) + .file(crtbegin_src) + .file(crtend_src); + + // Those flags are defined in src/llvm-project/compiler-rt/lib/crt/CMakeLists.txt + // Currently only consumer of those objects is musl, which use .init_array/.fini_array + // instead of .ctors/.dtors + cfg.flag("-std=c11") + .define("CRT_HAS_INITFINI_ARRAY", None) + .define("EH_USE_FRAME_REGISTRY", None); + + cfg.compile("crt"); + + t!(fs::copy(out_dir.join("crtbegin.o"), out_dir.join("crtbeginS.o"))); + t!(fs::copy(out_dir.join("crtend.o"), out_dir.join("crtendS.o"))); + out_dir + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Libunwind { + pub target: TargetSelection, +} + +impl Step for Libunwind { + type Output = PathBuf; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/llvm-project/libunwind") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Libunwind { target: run.target }); + } + + /// Build libunwind.a + fn run(self, builder: &Builder<'_>) -> Self::Output { + builder.update_submodule(&Path::new("src/llvm-project")); + + if builder.config.dry_run() { + return PathBuf::new(); + } + + let out_dir = builder.native_dir(self.target).join("libunwind"); + let root = builder.src.join("src/llvm-project/libunwind"); + + if up_to_date(&root, &out_dir.join("libunwind.a")) { + return out_dir; + } + + let _guard = builder.msg_unstaged(Kind::Build, "libunwind.a", self.target); + t!(fs::create_dir_all(&out_dir)); + + let mut cc_cfg = cc::Build::new(); + let mut cpp_cfg = cc::Build::new(); + + cpp_cfg.cpp(true); + cpp_cfg.cpp_set_stdlib(None); + cpp_cfg.flag("-nostdinc++"); + cpp_cfg.flag("-fno-exceptions"); + cpp_cfg.flag("-fno-rtti"); + cpp_cfg.flag_if_supported("-fvisibility-global-new-delete-hidden"); + + for cfg in [&mut cc_cfg, &mut cpp_cfg].iter_mut() { + if let Some(ar) = builder.ar(self.target) { + cfg.archiver(ar); + } + cfg.target(&self.target.triple); + cfg.host(&builder.config.build.triple); + cfg.warnings(false); + cfg.debug(false); + // get_compiler() need set opt_level first. + cfg.opt_level(3); + cfg.flag("-fstrict-aliasing"); + cfg.flag("-funwind-tables"); + cfg.flag("-fvisibility=hidden"); + cfg.define("_LIBUNWIND_DISABLE_VISIBILITY_ANNOTATIONS", None); + cfg.include(root.join("include")); + cfg.cargo_metadata(false); + cfg.out_dir(&out_dir); + + if self.target.contains("x86_64-fortanix-unknown-sgx") { + cfg.static_flag(true); + cfg.flag("-fno-stack-protector"); + cfg.flag("-ffreestanding"); + cfg.flag("-fexceptions"); + + // easiest way to undefine since no API available in cc::Build to undefine + cfg.flag("-U_FORTIFY_SOURCE"); + cfg.define("_FORTIFY_SOURCE", "0"); + cfg.define("RUST_SGX", "1"); + cfg.define("__NO_STRING_INLINES", None); + cfg.define("__NO_MATH_INLINES", None); + cfg.define("_LIBUNWIND_IS_BAREMETAL", None); + cfg.define("__LIBUNWIND_IS_NATIVE_ONLY", None); + cfg.define("NDEBUG", None); + } + if self.target.contains("windows") { + cfg.define("_LIBUNWIND_HIDE_SYMBOLS", "1"); + cfg.define("_LIBUNWIND_IS_NATIVE_ONLY", "1"); + } + } + + cc_cfg.compiler(builder.cc(self.target)); + if let Ok(cxx) = builder.cxx(self.target) { + cpp_cfg.compiler(cxx); + } else { + cc_cfg.compiler(builder.cc(self.target)); + } + + // Don't set this for clang + // By default, Clang builds C code in GNU C17 mode. + // By default, Clang builds C++ code according to the C++98 standard, + // with many C++11 features accepted as extensions. + if cc_cfg.get_compiler().is_like_gnu() { + cc_cfg.flag("-std=c99"); + } + if cpp_cfg.get_compiler().is_like_gnu() { + cpp_cfg.flag("-std=c++11"); + } + + if self.target.contains("x86_64-fortanix-unknown-sgx") || self.target.contains("musl") { + // use the same GCC C compiler command to compile C++ code so we do not need to setup the + // C++ compiler env variables on the builders. + // Don't set this for clang++, as clang++ is able to compile this without libc++. + if cpp_cfg.get_compiler().is_like_gnu() { + cpp_cfg.cpp(false); + cpp_cfg.compiler(builder.cc(self.target)); + } + } + + let mut c_sources = vec![ + "Unwind-sjlj.c", + "UnwindLevel1-gcc-ext.c", + "UnwindLevel1.c", + "UnwindRegistersRestore.S", + "UnwindRegistersSave.S", + ]; + + let cpp_sources = vec!["Unwind-EHABI.cpp", "Unwind-seh.cpp", "libunwind.cpp"]; + let cpp_len = cpp_sources.len(); + + if self.target.contains("x86_64-fortanix-unknown-sgx") { + c_sources.push("UnwindRustSgx.c"); + } + + for src in c_sources { + cc_cfg.file(root.join("src").join(src).canonicalize().unwrap()); + } + + for src in &cpp_sources { + cpp_cfg.file(root.join("src").join(src).canonicalize().unwrap()); + } + + cpp_cfg.compile("unwind-cpp"); + + // FIXME: https://github.com/alexcrichton/cc-rs/issues/545#issuecomment-679242845 + let mut count = 0; + for entry in fs::read_dir(&out_dir).unwrap() { + let file = entry.unwrap().path().canonicalize().unwrap(); + if file.is_file() && file.extension() == Some(OsStr::new("o")) { + // file name starts with "Unwind-EHABI", "Unwind-seh" or "libunwind" + let file_name = file.file_name().unwrap().to_str().expect("UTF-8 file name"); + if cpp_sources.iter().any(|f| file_name.starts_with(&f[..f.len() - 4])) { + cc_cfg.object(&file); + count += 1; + } + } + } + assert_eq!(cpp_len, count, "Can't get object files from {:?}", &out_dir); + + cc_cfg.compile("unwind"); + out_dir + } +} diff --git a/src/bootstrap/src/core/build_steps/mod.rs b/src/bootstrap/src/core/build_steps/mod.rs new file mode 100644 index 000000000..50d83789b --- /dev/null +++ b/src/bootstrap/src/core/build_steps/mod.rs @@ -0,0 +1,15 @@ +pub(crate) mod check; +pub(crate) mod clean; +pub(crate) mod compile; +pub(crate) mod dist; +pub(crate) mod doc; +pub(crate) mod format; +pub(crate) mod install; +pub(crate) mod llvm; +pub(crate) mod run; +pub(crate) mod setup; +pub(crate) mod suggest; +pub(crate) mod synthetic_targets; +pub(crate) mod test; +pub(crate) mod tool; +pub(crate) mod toolstate; diff --git a/src/bootstrap/src/core/build_steps/run.rs b/src/bootstrap/src/core/build_steps/run.rs new file mode 100644 index 000000000..d1d6b7e86 --- /dev/null +++ b/src/bootstrap/src/core/build_steps/run.rs @@ -0,0 +1,301 @@ +use std::path::PathBuf; +use std::process::Command; + +use crate::core::build_steps::dist::distdir; +use crate::core::build_steps::test; +use crate::core::build_steps::tool::{self, SourceType, Tool}; +use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::core::config::flags::get_completion; +use crate::core::config::TargetSelection; +use crate::utils::helpers::output; +use crate::Mode; + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct ExpandYamlAnchors; + +impl Step for ExpandYamlAnchors { + type Output = (); + + /// Runs the `expand-yaml_anchors` tool. + /// + /// This tool in `src/tools` reads the CI configuration files written in YAML and expands the + /// anchors in them, since GitHub Actions doesn't support them. + fn run(self, builder: &Builder<'_>) { + builder.info("Expanding YAML anchors in the GitHub Actions configuration"); + builder.run_delaying_failure( + &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("generate").arg(&builder.src), + ); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/expand-yaml-anchors") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(ExpandYamlAnchors); + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct BuildManifest; + +impl Step for BuildManifest { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/build-manifest") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(BuildManifest); + } + + fn run(self, builder: &Builder<'_>) { + // This gets called by `promote-release` + // (https://github.com/rust-lang/promote-release). + let mut cmd = builder.tool_cmd(Tool::BuildManifest); + let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| { + panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n") + }); + let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| { + panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n") + }); + + let today = output(Command::new("date").arg("+%Y-%m-%d")); + + cmd.arg(sign); + cmd.arg(distdir(builder)); + cmd.arg(today.trim()); + cmd.arg(addr); + cmd.arg(&builder.config.channel); + + builder.create_dir(&distdir(builder)); + builder.run(&mut cmd); + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct BumpStage0; + +impl Step for BumpStage0 { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/bump-stage0") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(BumpStage0); + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + let mut cmd = builder.tool_cmd(Tool::BumpStage0); + cmd.args(builder.config.args()); + builder.run(&mut cmd); + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct ReplaceVersionPlaceholder; + +impl Step for ReplaceVersionPlaceholder { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/replace-version-placeholder") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(ReplaceVersionPlaceholder); + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + let mut cmd = builder.tool_cmd(Tool::ReplaceVersionPlaceholder); + cmd.arg(&builder.src); + builder.run(&mut cmd); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Miri { + stage: u32, + host: TargetSelection, + target: TargetSelection, +} + +impl Step for Miri { + type Output = (); + const ONLY_HOSTS: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/miri") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Miri { + stage: run.builder.top_stage, + host: run.build_triple(), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let host = self.host; + let target = self.target; + let compiler = builder.compiler(stage, host); + + let miri = builder + .ensure(tool::Miri { compiler, target: self.host, extra_features: Vec::new() }) + .expect("in-tree tool"); + let miri_sysroot = test::Miri::build_miri_sysroot(builder, compiler, &miri, target); + + // # Run miri. + // Running it via `cargo run` as that figures out the right dylib path. + // add_rustc_lib_path does not add the path that contains librustc_driver-<...>.so. + let mut miri = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + host, + "run", + "src/tools/miri", + SourceType::InTree, + &[], + ); + miri.add_rustc_lib_path(builder, compiler); + // Forward arguments. + miri.arg("--").arg("--target").arg(target.rustc_target_arg()); + miri.args(builder.config.args()); + + // miri tests need to know about the stage sysroot + miri.env("MIRI_SYSROOT", &miri_sysroot); + + let mut miri = Command::from(miri); + builder.run(&mut miri); + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct CollectLicenseMetadata; + +impl Step for CollectLicenseMetadata { + type Output = PathBuf; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/collect-license-metadata") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(CollectLicenseMetadata); + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + let Some(reuse) = &builder.config.reuse else { + panic!("REUSE is required to collect the license metadata"); + }; + + // Temporary location, it will be moved to src/etc once it's accurate. + let dest = builder.out.join("license-metadata.json"); + + let mut cmd = builder.tool_cmd(Tool::CollectLicenseMetadata); + cmd.env("REUSE_EXE", reuse); + cmd.env("DEST", &dest); + builder.run(&mut cmd); + + dest + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct GenerateCopyright; + +impl Step for GenerateCopyright { + type Output = PathBuf; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/generate-copyright") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(GenerateCopyright); + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + let license_metadata = builder.ensure(CollectLicenseMetadata); + + // Temporary location, it will be moved to the proper one once it's accurate. + let dest = builder.out.join("COPYRIGHT.md"); + + let mut cmd = builder.tool_cmd(Tool::GenerateCopyright); + cmd.env("LICENSE_METADATA", &license_metadata); + cmd.env("DEST", &dest); + builder.run(&mut cmd); + + dest + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct GenerateWindowsSys; + +impl Step for GenerateWindowsSys { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/generate-windows-sys") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(GenerateWindowsSys); + } + + fn run(self, builder: &Builder<'_>) { + let mut cmd = builder.tool_cmd(Tool::GenerateWindowsSys); + cmd.arg(&builder.src); + builder.run(&mut cmd); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct GenerateCompletions; + +macro_rules! generate_completions { + ( $( ( $shell:ident, $filename:expr ) ),* ) => { + $( + if let Some(comp) = get_completion($shell, &$filename) { + std::fs::write(&$filename, comp).expect(&format!("writing {} completion", stringify!($shell))); + } + )* + }; +} + +impl Step for GenerateCompletions { + type Output = (); + + /// Uses `clap_complete` to generate shell completions. + fn run(self, builder: &Builder<'_>) { + use clap_complete::shells::{Bash, Fish, PowerShell, Zsh}; + + generate_completions!( + (Bash, builder.src.join("src/etc/completions/x.py.sh")), + (Zsh, builder.src.join("src/etc/completions/x.py.zsh")), + (Fish, builder.src.join("src/etc/completions/x.py.fish")), + (PowerShell, builder.src.join("src/etc/completions/x.py.ps1")) + ); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("generate-completions") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(GenerateCompletions); + } +} diff --git a/src/bootstrap/src/core/build_steps/setup.rs b/src/bootstrap/src/core/build_steps/setup.rs new file mode 100644 index 000000000..486a1e20f --- /dev/null +++ b/src/bootstrap/src/core/build_steps/setup.rs @@ -0,0 +1,623 @@ +use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::Config; +use crate::{t, CONFIG_CHANGE_HISTORY}; +use sha2::Digest; +use std::env::consts::EXE_SUFFIX; +use std::fmt::Write as _; +use std::fs::File; +use std::io::Write; +use std::path::{Path, PathBuf, MAIN_SEPARATOR}; +use std::process::Command; +use std::str::FromStr; +use std::{fmt, fs, io}; + +#[cfg(test)] +#[path = "../../tests/setup.rs"] +mod tests; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub enum Profile { + Compiler, + Codegen, + Library, + Tools, + Dist, + None, +} + +/// A list of historical hashes of `src/etc/rust_analyzer_settings.json`. +/// New entries should be appended whenever this is updated so we can detect +/// outdated vs. user-modified settings files. +static SETTINGS_HASHES: &[&str] = &[ + "ea67e259dedf60d4429b6c349a564ffcd1563cf41c920a856d1f5b16b4701ac8", + "56e7bf011c71c5d81e0bf42e84938111847a810eee69d906bba494ea90b51922", + "af1b5efe196aed007577899db9dae15d6dbc923d6fa42fa0934e68617ba9bbe0", + "3468fea433c25fff60be6b71e8a215a732a7b1268b6a83bf10d024344e140541", + "47d227f424bf889b0d899b9cc992d5695e1b78c406e183cd78eafefbe5488923", + "b526bd58d0262dd4dda2bff5bc5515b705fb668a46235ace3e057f807963a11a", +]; +static RUST_ANALYZER_SETTINGS: &str = include_str!("../../../../etc/rust_analyzer_settings.json"); + +impl Profile { + fn include_path(&self, src_path: &Path) -> PathBuf { + PathBuf::from(format!("{}/src/bootstrap/defaults/config.{}.toml", src_path.display(), self)) + } + + pub fn all() -> impl Iterator { + use Profile::*; + // N.B. these are ordered by how they are displayed, not alphabetically + [Library, Compiler, Codegen, Tools, Dist, None].iter().copied() + } + + pub fn purpose(&self) -> String { + use Profile::*; + match self { + Library => "Contribute to the standard library", + Compiler => "Contribute to the compiler itself", + Codegen => "Contribute to the compiler, and also modify LLVM or codegen", + Tools => "Contribute to tools which depend on the compiler, but do not modify it directly (e.g. rustdoc, clippy, miri)", + Dist => "Install Rust from source", + None => "Do not modify `config.toml`" + } + .to_string() + } + + pub fn all_for_help(indent: &str) -> String { + let mut out = String::new(); + for choice in Profile::all() { + writeln!(&mut out, "{}{}: {}", indent, choice, choice.purpose()).unwrap(); + } + out + } + + pub fn as_str(&self) -> &'static str { + match self { + Profile::Compiler => "compiler", + Profile::Codegen => "codegen", + Profile::Library => "library", + Profile::Tools => "tools", + Profile::Dist => "dist", + Profile::None => "none", + } + } +} + +impl FromStr for Profile { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "lib" | "library" => Ok(Profile::Library), + "compiler" => Ok(Profile::Compiler), + "llvm" | "codegen" => Ok(Profile::Codegen), + "maintainer" | "dist" | "user" => Ok(Profile::Dist), + "tools" | "tool" | "rustdoc" | "clippy" | "miri" | "rustfmt" | "rls" => { + Ok(Profile::Tools) + } + "none" => Ok(Profile::None), + _ => Err(format!("unknown profile: '{s}'")), + } + } +} + +impl fmt::Display for Profile { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl Step for Profile { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(mut run: ShouldRun<'_>) -> ShouldRun<'_> { + for choice in Profile::all() { + run = run.alias(choice.as_str()); + } + run + } + + fn make_run(run: RunConfig<'_>) { + if run.builder.config.dry_run() { + return; + } + + let path = &run.builder.config.config.clone().unwrap_or(PathBuf::from("config.toml")); + if path.exists() { + eprintln!(); + eprintln!( + "ERROR: you asked for a new config file, but one already exists at `{}`", + t!(path.canonicalize()).display() + ); + + match prompt_user( + "Do you wish to override the existing configuration (which will allow the setup process to continue)?: [y/N]", + ) { + Ok(Some(PromptResult::Yes)) => { + t!(fs::remove_file(path)); + } + _ => { + println!("Exiting."); + crate::exit!(1); + } + } + } + + // for Profile, `run.paths` will have 1 and only 1 element + // this is because we only accept at most 1 path from user input. + // If user calls `x.py setup` without arguments, the interactive TUI + // will guide user to provide one. + let profile = if run.paths.len() > 1 { + // HACK: `builder` runs this step with all paths if no path was passed. + t!(interactive_path()) + } else { + run.paths + .first() + .unwrap() + .assert_single_path() + .path + .as_path() + .as_os_str() + .to_str() + .unwrap() + .parse() + .unwrap() + }; + + run.builder.ensure(profile); + } + + fn run(self, builder: &Builder<'_>) { + // During ./x.py setup once you select the codegen profile. + // The submodule will be downloaded. It does not work in the + // tarball case since they don't include Git and submodules + // are already included. + if !builder.rust_info().is_from_tarball() { + if self == Profile::Codegen { + builder.update_submodule(&Path::new("src/llvm-project")); + } + } + setup(&builder.build.config, self) + } +} + +pub fn setup(config: &Config, profile: Profile) { + let suggestions: &[&str] = match profile { + Profile::Codegen | Profile::Compiler | Profile::None => &["check", "build", "test"], + Profile::Tools => &[ + "check", + "build", + "test tests/rustdoc*", + "test src/tools/clippy", + "test src/tools/miri", + "test src/tools/rustfmt", + ], + Profile::Library => &["check", "build", "test library/std", "doc"], + Profile::Dist => &["dist", "build"], + }; + + println!(); + + println!("To get started, try one of the following commands:"); + for cmd in suggestions { + println!("- `x.py {cmd}`"); + } + + if profile != Profile::Dist { + println!( + "For more suggestions, see https://rustc-dev-guide.rust-lang.org/building/suggested.html" + ); + } + + if profile == Profile::Tools { + eprintln!(); + eprintln!( + "NOTE: the `tools` profile sets up the `stage2` toolchain (use \ + `rustup toolchain link 'name' build/host/stage2` to use rustc)" + ) + } + + let path = &config.config.clone().unwrap_or(PathBuf::from("config.toml")); + setup_config_toml(path, profile, config); +} + +fn setup_config_toml(path: &PathBuf, profile: Profile, config: &Config) { + if profile == Profile::None { + return; + } + + let latest_change_id = CONFIG_CHANGE_HISTORY.last().unwrap(); + let settings = format!( + "# Includes one of the default files in src/bootstrap/defaults\n\ + profile = \"{profile}\"\n\ + change-id = {latest_change_id}\n" + ); + + t!(fs::write(path, settings)); + + let include_path = profile.include_path(&config.src); + println!("`x.py` will now use the configuration at {}", include_path.display()); +} + +/// Creates a toolchain link for stage1 using `rustup` +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub struct Link; +impl Step for Link { + type Output = (); + const DEFAULT: bool = true; + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("link") + } + fn make_run(run: RunConfig<'_>) { + if run.builder.config.dry_run() { + return; + } + if let [cmd] = &run.paths[..] { + if cmd.assert_single_path().path.as_path().as_os_str() == "link" { + run.builder.ensure(Link); + } + } + } + fn run(self, builder: &Builder<'_>) -> Self::Output { + let config = &builder.config; + if config.dry_run() { + return; + } + let stage_path = + ["build", config.build.rustc_target_arg(), "stage1"].join(&MAIN_SEPARATOR.to_string()); + + if !rustup_installed() { + eprintln!("`rustup` is not installed; cannot link `stage1` toolchain"); + } else if stage_dir_exists(&stage_path[..]) && !config.dry_run() { + attempt_toolchain_link(&stage_path[..]); + } + } +} + +fn rustup_installed() -> bool { + Command::new("rustup") + .arg("--version") + .stdout(std::process::Stdio::null()) + .output() + .map_or(false, |output| output.status.success()) +} + +fn stage_dir_exists(stage_path: &str) -> bool { + match fs::create_dir(&stage_path) { + Ok(_) => true, + Err(_) => Path::new(&stage_path).exists(), + } +} + +fn attempt_toolchain_link(stage_path: &str) { + if toolchain_is_linked() { + return; + } + + if !ensure_stage1_toolchain_placeholder_exists(stage_path) { + eprintln!( + "Failed to create a template for stage 1 toolchain or confirm that it already exists" + ); + return; + } + + if try_link_toolchain(&stage_path) { + println!( + "Added `stage1` rustup toolchain; try `cargo +stage1 build` on a separate rust project to run a newly-built toolchain" + ); + } else { + eprintln!("`rustup` failed to link stage 1 build to `stage1` toolchain"); + eprintln!( + "To manually link stage 1 build to `stage1` toolchain, run:\n + `rustup toolchain link stage1 {}`", + &stage_path + ); + } +} + +fn toolchain_is_linked() -> bool { + match Command::new("rustup") + .args(&["toolchain", "list"]) + .stdout(std::process::Stdio::piped()) + .output() + { + Ok(toolchain_list) => { + if !String::from_utf8_lossy(&toolchain_list.stdout).contains("stage1") { + return false; + } + // The toolchain has already been linked. + println!( + "`stage1` toolchain already linked; not attempting to link `stage1` toolchain" + ); + } + Err(_) => { + // In this case, we don't know if the `stage1` toolchain has been linked; + // but `rustup` failed, so let's not go any further. + println!( + "`rustup` failed to list current toolchains; not attempting to link `stage1` toolchain" + ); + } + } + true +} + +fn try_link_toolchain(stage_path: &str) -> bool { + Command::new("rustup") + .stdout(std::process::Stdio::null()) + .args(&["toolchain", "link", "stage1", &stage_path]) + .output() + .map_or(false, |output| output.status.success()) +} + +fn ensure_stage1_toolchain_placeholder_exists(stage_path: &str) -> bool { + let pathbuf = PathBuf::from(stage_path); + + if fs::create_dir_all(pathbuf.join("lib")).is_err() { + return false; + }; + + let pathbuf = pathbuf.join("bin"); + if fs::create_dir_all(&pathbuf).is_err() { + return false; + }; + + let pathbuf = pathbuf.join(format!("rustc{EXE_SUFFIX}")); + + if pathbuf.exists() { + return true; + } + + // Take care not to overwrite the file + let result = File::options().append(true).create(true).open(&pathbuf); + if result.is_err() { + return false; + } + + return true; +} + +// Used to get the path for `Subcommand::Setup` +pub fn interactive_path() -> io::Result { + fn abbrev_all() -> impl Iterator { + ('a'..) + .zip(1..) + .map(|(letter, number)| (letter.to_string(), number.to_string())) + .zip(Profile::all()) + } + + fn parse_with_abbrev(input: &str) -> Result { + let input = input.trim().to_lowercase(); + for ((letter, number), profile) in abbrev_all() { + if input == letter || input == number { + return Ok(profile); + } + } + input.parse() + } + + println!("Welcome to the Rust project! What do you want to do with x.py?"); + for ((letter, _), profile) in abbrev_all() { + println!("{}) {}: {}", letter, profile, profile.purpose()); + } + let template = loop { + print!( + "Please choose one ({}): ", + abbrev_all().map(|((l, _), _)| l).collect::>().join("/") + ); + io::stdout().flush()?; + let mut input = String::new(); + io::stdin().read_line(&mut input)?; + if input.is_empty() { + eprintln!("EOF on stdin, when expecting answer to question. Giving up."); + crate::exit!(1); + } + break match parse_with_abbrev(&input) { + Ok(profile) => profile, + Err(err) => { + eprintln!("ERROR: {err}"); + eprintln!("NOTE: press Ctrl+C to exit"); + continue; + } + }; + }; + Ok(template) +} + +#[derive(PartialEq)] +enum PromptResult { + Yes, // y/Y/yes + No, // n/N/no + Print, // p/P/print +} + +/// Prompt a user for a answer, looping until they enter an accepted input or nothing +fn prompt_user(prompt: &str) -> io::Result> { + let mut input = String::new(); + loop { + print!("{prompt} "); + io::stdout().flush()?; + input.clear(); + io::stdin().read_line(&mut input)?; + match input.trim().to_lowercase().as_str() { + "y" | "yes" => return Ok(Some(PromptResult::Yes)), + "n" | "no" => return Ok(Some(PromptResult::No)), + "p" | "print" => return Ok(Some(PromptResult::Print)), + "" => return Ok(None), + _ => { + eprintln!("ERROR: unrecognized option '{}'", input.trim()); + eprintln!("NOTE: press Ctrl+C to exit"); + } + }; + } +} + +/// Installs `src/etc/pre-push.sh` as a Git hook +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub struct Hook; + +impl Step for Hook { + type Output = (); + const DEFAULT: bool = true; + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("hook") + } + fn make_run(run: RunConfig<'_>) { + if run.builder.config.dry_run() { + return; + } + if let [cmd] = &run.paths[..] { + if cmd.assert_single_path().path.as_path().as_os_str() == "hook" { + run.builder.ensure(Hook); + } + } + } + fn run(self, builder: &Builder<'_>) -> Self::Output { + let config = &builder.config; + if config.dry_run() { + return; + } + t!(install_git_hook_maybe(&config)); + } +} + +// install a git hook to automatically run tidy, if they want +fn install_git_hook_maybe(config: &Config) -> io::Result<()> { + let git = t!(config.git().args(&["rev-parse", "--git-common-dir"]).output().map(|output| { + assert!(output.status.success(), "failed to run `git`"); + PathBuf::from(t!(String::from_utf8(output.stdout)).trim()) + })); + let hooks_dir = git.join("hooks"); + let dst = hooks_dir.join("pre-push"); + if dst.exists() { + // The git hook has already been set up, or the user already has a custom hook. + return Ok(()); + } + + println!( + "\nRust's CI will automatically fail if it doesn't pass `tidy`, the internal tool for ensuring code quality. +If you'd like, x.py can install a git hook for you that will automatically run `test tidy` before +pushing your code to ensure your code is up to par. If you decide later that this behavior is +undesirable, simply delete the `pre-push` file from .git/hooks." + ); + + if prompt_user("Would you like to install the git hook?: [y/N]")? != Some(PromptResult::Yes) { + println!("Ok, skipping installation!"); + return Ok(()); + } + if !hooks_dir.exists() { + // We need to (try to) create the hooks directory first. + let _ = fs::create_dir(hooks_dir); + } + let src = config.src.join("src").join("etc").join("pre-push.sh"); + match fs::hard_link(src, &dst) { + Err(e) => { + eprintln!( + "ERROR: could not create hook {}: do you already have the git hook installed?\n{}", + dst.display(), + e + ); + return Err(e); + } + Ok(_) => println!("Linked `src/etc/pre-push.sh` to `.git/hooks/pre-push`"), + }; + Ok(()) +} + +/// Sets up or displays `src/etc/rust_analyzer_settings.json` +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub struct Vscode; + +impl Step for Vscode { + type Output = (); + const DEFAULT: bool = true; + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("vscode") + } + fn make_run(run: RunConfig<'_>) { + if run.builder.config.dry_run() { + return; + } + if let [cmd] = &run.paths[..] { + if cmd.assert_single_path().path.as_path().as_os_str() == "vscode" { + run.builder.ensure(Vscode); + } + } + } + fn run(self, builder: &Builder<'_>) -> Self::Output { + let config = &builder.config; + if config.dry_run() { + return; + } + t!(create_vscode_settings_maybe(&config)); + } +} + +/// Create a `.vscode/settings.json` file for rustc development, or just print it +fn create_vscode_settings_maybe(config: &Config) -> io::Result<()> { + let (current_hash, historical_hashes) = SETTINGS_HASHES.split_last().unwrap(); + let vscode_settings = config.src.join(".vscode").join("settings.json"); + // If None, no settings.json exists + // If Some(true), is a previous version of settings.json + // If Some(false), is not a previous version (i.e. user modified) + // If it's up to date we can just skip this + let mut mismatched_settings = None; + if let Ok(current) = fs::read_to_string(&vscode_settings) { + let mut hasher = sha2::Sha256::new(); + hasher.update(¤t); + let hash = hex::encode(hasher.finalize().as_slice()); + if hash == *current_hash { + return Ok(()); + } else if historical_hashes.contains(&hash.as_str()) { + mismatched_settings = Some(true); + } else { + mismatched_settings = Some(false); + } + } + println!( + "\nx.py can automatically install the recommended `.vscode/settings.json` file for rustc development" + ); + match mismatched_settings { + Some(true) => eprintln!( + "WARNING: existing `.vscode/settings.json` is out of date, x.py will update it" + ), + Some(false) => eprintln!( + "WARNING: existing `.vscode/settings.json` has been modified by user, x.py will back it up and replace it" + ), + _ => (), + } + let should_create = match prompt_user( + "Would you like to create/update `settings.json`, or only print suggested settings?: [y/p/N]", + )? { + Some(PromptResult::Yes) => true, + Some(PromptResult::Print) => false, + _ => { + println!("Ok, skipping settings!"); + return Ok(()); + } + }; + if should_create { + let path = config.src.join(".vscode"); + if !path.exists() { + fs::create_dir(&path)?; + } + let verb = match mismatched_settings { + // exists but outdated, we can replace this + Some(true) => "Updated", + // exists but user modified, back it up + Some(false) => { + // exists and is not current version or outdated, so back it up + let mut backup = vscode_settings.clone(); + backup.set_extension("json.bak"); + eprintln!("WARNING: copying `settings.json` to `settings.json.bak`"); + fs::copy(&vscode_settings, &backup)?; + "Updated" + } + _ => "Created", + }; + fs::write(&vscode_settings, &RUST_ANALYZER_SETTINGS)?; + println!("{verb} `.vscode/settings.json`"); + } else { + println!("\n{RUST_ANALYZER_SETTINGS}"); + } + Ok(()) +} diff --git a/src/bootstrap/src/core/build_steps/suggest.rs b/src/bootstrap/src/core/build_steps/suggest.rs new file mode 100644 index 000000000..93da27560 --- /dev/null +++ b/src/bootstrap/src/core/build_steps/suggest.rs @@ -0,0 +1,78 @@ +#![cfg_attr(feature = "build-metrics", allow(unused))] + +use clap::Parser; +use std::path::PathBuf; +use std::str::FromStr; + +use crate::core::build_steps::tool::Tool; +use crate::core::builder::Builder; + +/// Suggests a list of possible `x.py` commands to run based on modified files in branch. +pub fn suggest(builder: &Builder<'_>, run: bool) { + let git_config = builder.config.git_config(); + let suggestions = builder + .tool_cmd(Tool::SuggestTests) + .env("SUGGEST_TESTS_GIT_REPOSITORY", git_config.git_repository) + .env("SUGGEST_TESTS_NIGHTLY_BRANCH", git_config.nightly_branch) + .output() + .expect("failed to run `suggest-tests` tool"); + + if !suggestions.status.success() { + println!("failed to run `suggest-tests` tool ({})", suggestions.status); + println!( + "`suggest_tests` stdout:\n{}`suggest_tests` stderr:\n{}", + String::from_utf8(suggestions.stdout).unwrap(), + String::from_utf8(suggestions.stderr).unwrap() + ); + panic!("failed to run `suggest-tests`"); + } + + let suggestions = String::from_utf8(suggestions.stdout).unwrap(); + let suggestions = suggestions + .lines() + .map(|line| { + let mut sections = line.split_ascii_whitespace(); + + // this code expects one suggestion per line in the following format: + // {some number of flags} [optional stage number] + let cmd = sections.next().unwrap(); + let stage = sections.next_back().map(|s| str::parse(s).ok()).flatten(); + let paths: Vec = sections.map(|p| PathBuf::from_str(p).unwrap()).collect(); + + (cmd, stage, paths) + }) + .collect::>(); + + if !suggestions.is_empty() { + println!("==== SUGGESTIONS ===="); + for sug in &suggestions { + print!("x {} ", sug.0); + if let Some(stage) = sug.1 { + print!("--stage {stage} "); + } + + for path in &sug.2 { + print!("{} ", path.display()); + } + println!(); + } + println!("====================="); + } else { + println!("No suggestions found!"); + return; + } + + if run { + for sug in suggestions { + let mut build: crate::Build = builder.build.clone(); + build.config.paths = sug.2; + build.config.cmd = crate::core::config::flags::Flags::parse_from(["x.py", sug.0]).cmd; + if let Some(stage) = sug.1 { + build.config.stage = stage; + } + build.build(); + } + } else { + println!("HELP: consider using the `--run` flag to automatically run suggested tests"); + } +} diff --git a/src/bootstrap/src/core/build_steps/synthetic_targets.rs b/src/bootstrap/src/core/build_steps/synthetic_targets.rs new file mode 100644 index 000000000..d2c65b740 --- /dev/null +++ b/src/bootstrap/src/core/build_steps/synthetic_targets.rs @@ -0,0 +1,82 @@ +//! In some cases, parts of bootstrap need to change part of a target spec just for one or a few +//! steps. Adding these targets to rustc proper would "leak" this implementation detail of +//! bootstrap, and would make it more complex to apply additional changes if the need arises. +//! +//! To address that problem, this module implements support for "synthetic targets". Synthetic +//! targets are custom target specs generated using builtin target specs as their base. You can use +//! one of the target specs already defined in this module, or create new ones by adding a new step +//! that calls create_synthetic_target. + +use crate::core::builder::{Builder, ShouldRun, Step}; +use crate::core::config::TargetSelection; +use crate::Compiler; +use std::process::{Command, Stdio}; + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub(crate) struct MirOptPanicAbortSyntheticTarget { + pub(crate) compiler: Compiler, + pub(crate) base: TargetSelection, +} + +impl Step for MirOptPanicAbortSyntheticTarget { + type Output = TargetSelection; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + create_synthetic_target(builder, self.compiler, "miropt-abort", self.base, |spec| { + spec.insert("panic-strategy".into(), "abort".into()); + }) + } +} + +fn create_synthetic_target( + builder: &Builder<'_>, + compiler: Compiler, + suffix: &str, + base: TargetSelection, + customize: impl FnOnce(&mut serde_json::Map), +) -> TargetSelection { + if base.contains("synthetic") { + // This check is not strictly needed, but nothing currently needs recursive synthetic + // targets. If the need arises, removing this in the future *SHOULD* be safe. + panic!("cannot create synthetic targets with other synthetic targets as their base"); + } + + let name = format!("{base}-synthetic-{suffix}"); + let path = builder.out.join("synthetic-target-specs").join(format!("{name}.json")); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + + if builder.config.dry_run() { + std::fs::write(&path, b"dry run\n").unwrap(); + return TargetSelection::create_synthetic(&name, path.to_str().unwrap()); + } + + let mut cmd = Command::new(builder.rustc(compiler)); + cmd.arg("--target").arg(base.rustc_target_arg()); + cmd.args(["-Zunstable-options", "--print", "target-spec-json"]); + cmd.stdout(Stdio::piped()); + + let output = cmd.spawn().unwrap().wait_with_output().unwrap(); + if !output.status.success() { + panic!("failed to gather the target spec for {base}"); + } + + let mut spec: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + let spec_map = spec.as_object_mut().unwrap(); + + // The `is-builtin` attribute of a spec needs to be removed, otherwise rustc will complain. + spec_map.remove("is-builtin"); + + customize(spec_map); + + std::fs::write(&path, &serde_json::to_vec_pretty(&spec).unwrap()).unwrap(); + let target = TargetSelection::create_synthetic(&name, path.to_str().unwrap()); + crate::utils::cc_detect::find_target(builder, target); + + target +} diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs new file mode 100644 index 000000000..d2aa89dee --- /dev/null +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -0,0 +1,3315 @@ +//! Implementation of the test-related targets of the build system. +//! +//! This file implements the various regression test suites that we execute on +//! our CI. + +use std::env; +use std::ffi::OsStr; +use std::ffi::OsString; +use std::fs; +use std::iter; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +use clap_complete::shells; + +use crate::core::build_steps::compile; +use crate::core::build_steps::dist; +use crate::core::build_steps::doc::DocumentationFormat; +use crate::core::build_steps::llvm; +use crate::core::build_steps::synthetic_targets::MirOptPanicAbortSyntheticTarget; +use crate::core::build_steps::tool::{self, SourceType, Tool}; +use crate::core::build_steps::toolstate::ToolState; +use crate::core::builder::crate_description; +use crate::core::builder::{Builder, Compiler, Kind, RunConfig, ShouldRun, Step}; +use crate::core::config::flags::get_completion; +use crate::core::config::flags::Subcommand; +use crate::core::config::TargetSelection; +use crate::utils; +use crate::utils::cache::{Interned, INTERNER}; +use crate::utils::exec::BootstrapCommand; +use crate::utils::helpers::{ + self, add_link_lib_path, dylib_path, dylib_path_var, output, t, + target_supports_cranelift_backend, up_to_date, +}; +use crate::utils::render_tests::{add_flags_and_try_run_tests, try_run_tests}; +use crate::{envify, CLang, DocTests, GitRepo, Mode}; + +const ADB_TEST_DIR: &str = "/data/local/tmp/work"; + +// mir-opt tests have different variants depending on whether a target is 32bit or 64bit, and +// blessing them requires blessing with each target. To aid developers, when blessing the mir-opt +// test suite the corresponding target of the opposite pointer size is also blessed. +// +// This array serves as the known mappings between 32bit and 64bit targets. If you're developing on +// a target where a target with the opposite pointer size exists, feel free to add it here. +const MIR_OPT_BLESS_TARGET_MAPPING: &[(&str, &str)] = &[ + // (32bit, 64bit) + ("i686-unknown-linux-gnu", "x86_64-unknown-linux-gnu"), + ("i686-unknown-linux-musl", "x86_64-unknown-linux-musl"), + ("i686-pc-windows-msvc", "x86_64-pc-windows-msvc"), + ("i686-pc-windows-gnu", "x86_64-pc-windows-gnu"), + ("i686-apple-darwin", "x86_64-apple-darwin"), + // ARM Macs don't have a corresponding 32-bit target that they can (easily) + // build for, so there is no entry for "aarch64-apple-darwin" here. +]; + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct CrateBootstrap { + path: Interned, + host: TargetSelection, +} + +impl Step for CrateBootstrap { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/jsondoclint") + .path("src/tools/suggest-tests") + .path("src/tools/replace-version-placeholder") + .alias("tidyselftest") + } + + fn make_run(run: RunConfig<'_>) { + for path in run.paths { + let path = INTERNER.intern_path(path.assert_single_path().path.clone()); + run.builder.ensure(CrateBootstrap { host: run.target, path }); + } + } + + fn run(self, builder: &Builder<'_>) { + let bootstrap_host = builder.config.build; + let compiler = builder.compiler(0, bootstrap_host); + let mut path = self.path.to_str().unwrap(); + if path == "tidyselftest" { + path = "src/tools/tidy"; + } + + let cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolBootstrap, + bootstrap_host, + "test", + path, + SourceType::InTree, + &[], + ); + let crate_name = path.rsplit_once('/').unwrap().1; + run_cargo_test(cargo, &[], &[], crate_name, crate_name, compiler, bootstrap_host, builder); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Linkcheck { + host: TargetSelection, +} + +impl Step for Linkcheck { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + /// Runs the `linkchecker` tool as compiled in `stage` by the `host` compiler. + /// + /// This tool in `src/tools` will verify the validity of all our links in the + /// documentation to ensure we don't have a bunch of dead ones. + fn run(self, builder: &Builder<'_>) { + let host = self.host; + let hosts = &builder.hosts; + let targets = &builder.targets; + + // if we have different hosts and targets, some things may be built for + // the host (e.g. rustc) and others for the target (e.g. std). The + // documentation built for each will contain broken links to + // docs built for the other platform (e.g. rustc linking to cargo) + if (hosts != targets) && !hosts.is_empty() && !targets.is_empty() { + panic!( + "Linkcheck currently does not support builds with different hosts and targets. +You can skip linkcheck with --skip src/tools/linkchecker" + ); + } + + builder.info(&format!("Linkcheck ({host})")); + + // Test the linkchecker itself. + let bootstrap_host = builder.config.build; + let compiler = builder.compiler(0, bootstrap_host); + + let cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolBootstrap, + bootstrap_host, + "test", + "src/tools/linkchecker", + SourceType::InTree, + &[], + ); + run_cargo_test( + cargo, + &[], + &[], + "linkchecker", + "linkchecker self tests", + compiler, + bootstrap_host, + builder, + ); + + if builder.doc_tests == DocTests::No { + return; + } + + // Build all the default documentation. + builder.default_doc(&[]); + + // Build the linkchecker before calling `msg`, since GHA doesn't support nested groups. + let mut linkchecker = builder.tool_cmd(Tool::Linkchecker); + + // Run the linkchecker. + let _guard = + builder.msg(Kind::Test, compiler.stage, "Linkcheck", bootstrap_host, bootstrap_host); + let _time = helpers::timeit(&builder); + builder.run_delaying_failure(linkchecker.arg(builder.out.join(host.triple).join("doc"))); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + let run = run.path("src/tools/linkchecker"); + run.default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Linkcheck { host: run.target }); + } +} + +fn check_if_tidy_is_installed() -> bool { + Command::new("tidy") + .arg("--version") + .stdout(Stdio::null()) + .status() + .map_or(false, |status| status.success()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct HtmlCheck { + target: TargetSelection, +} + +impl Step for HtmlCheck { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let run = run.path("src/tools/html-checker"); + run.lazy_default_condition(Box::new(check_if_tidy_is_installed)) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(HtmlCheck { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + if !check_if_tidy_is_installed() { + eprintln!("not running HTML-check tool because `tidy` is missing"); + eprintln!( + "Note that `tidy` is not the in-tree `src/tools/tidy` but needs to be installed" + ); + panic!("Cannot run html-check tests"); + } + // Ensure that a few different kinds of documentation are available. + builder.default_doc(&[]); + builder.ensure(crate::core::build_steps::doc::Rustc::new( + builder.top_stage, + self.target, + builder, + )); + + builder.run_delaying_failure( + builder.tool_cmd(Tool::HtmlChecker).arg(builder.doc_out(self.target)), + ); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Cargotest { + stage: u32, + host: TargetSelection, +} + +impl Step for Cargotest { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/cargotest") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Cargotest { stage: run.builder.top_stage, host: run.target }); + } + + /// Runs the `cargotest` tool as compiled in `stage` by the `host` compiler. + /// + /// This tool in `src/tools` will check out a few Rust projects and run `cargo + /// test` to ensure that we don't regress the test suites there. + fn run(self, builder: &Builder<'_>) { + let compiler = builder.compiler(self.stage, self.host); + builder.ensure(compile::Rustc::new(compiler, compiler.host)); + let cargo = builder.ensure(tool::Cargo { compiler, target: compiler.host }); + + // Note that this is a short, cryptic, and not scoped directory name. This + // is currently to minimize the length of path on Windows where we otherwise + // quickly run into path name limit constraints. + let out_dir = builder.out.join("ct"); + t!(fs::create_dir_all(&out_dir)); + + let _time = helpers::timeit(&builder); + let mut cmd = builder.tool_cmd(Tool::CargoTest); + builder.run_delaying_failure( + cmd.arg(&cargo) + .arg(&out_dir) + .args(builder.config.test_args()) + .env("RUSTC", builder.rustc(compiler)) + .env("RUSTDOC", builder.rustdoc(compiler)), + ); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Cargo { + stage: u32, + host: TargetSelection, +} + +impl Step for Cargo { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/cargo") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Cargo { stage: run.builder.top_stage, host: run.target }); + } + + /// Runs `cargo test` for `cargo` packaged with Rust. + fn run(self, builder: &Builder<'_>) { + let compiler = builder.compiler(self.stage, self.host); + + builder.ensure(tool::Cargo { compiler, target: self.host }); + let cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + self.host, + "test", + "src/tools/cargo", + SourceType::Submodule, + &[], + ); + + // NOTE: can't use `run_cargo_test` because we need to overwrite `PATH` + let mut cargo = prepare_cargo_test(cargo, &[], &[], "cargo", compiler, self.host, builder); + + // Don't run cross-compile tests, we may not have cross-compiled libstd libs + // available. + cargo.env("CFG_DISABLE_CROSS_TESTS", "1"); + // Forcibly disable tests using nightly features since any changes to + // those features won't be able to land. + cargo.env("CARGO_TEST_DISABLE_NIGHTLY", "1"); + cargo.env("PATH", &path_for_cargo(builder, compiler)); + + #[cfg(feature = "build-metrics")] + builder.metrics.begin_test_suite( + build_helper::metrics::TestSuiteMetadata::CargoPackage { + crates: vec!["cargo".into()], + target: self.host.triple.to_string(), + host: self.host.triple.to_string(), + stage: self.stage, + }, + builder, + ); + + let _time = helpers::timeit(&builder); + add_flags_and_try_run_tests(builder, &mut cargo); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct RustAnalyzer { + stage: u32, + host: TargetSelection, +} + +impl Step for RustAnalyzer { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rust-analyzer") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Self { stage: run.builder.top_stage, host: run.target }); + } + + /// Runs `cargo test` for rust-analyzer + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let host = self.host; + let compiler = builder.compiler(stage, host); + + // We don't need to build the whole Rust Analyzer for the proc-macro-srv test suite, + // but we do need the standard library to be present. + builder.ensure(compile::Std::new(compiler, host)); + + let workspace_path = "src/tools/rust-analyzer"; + // until the whole RA test suite runs on `i686`, we only run + // `proc-macro-srv` tests + let crate_path = "src/tools/rust-analyzer/crates/proc-macro-srv"; + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolStd, + host, + "test", + crate_path, + SourceType::InTree, + &["sysroot-abi".to_owned()], + ); + cargo.allow_features(tool::RustAnalyzer::ALLOW_FEATURES); + + let dir = builder.src.join(workspace_path); + // needed by rust-analyzer to find its own text fixtures, cf. + // https://github.com/rust-analyzer/expect-test/issues/33 + cargo.env("CARGO_WORKSPACE_DIR", &dir); + + // RA's test suite tries to write to the source directory, that can't + // work in Rust CI + cargo.env("SKIP_SLOW_TESTS", "1"); + + cargo.add_rustc_lib_path(builder, compiler); + run_cargo_test(cargo, &[], &[], "rust-analyzer", "rust-analyzer", compiler, host, builder); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Rustfmt { + stage: u32, + host: TargetSelection, +} + +impl Step for Rustfmt { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rustfmt") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Rustfmt { stage: run.builder.top_stage, host: run.target }); + } + + /// Runs `cargo test` for rustfmt. + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let host = self.host; + let compiler = builder.compiler(stage, host); + + builder + .ensure(tool::Rustfmt { compiler, target: self.host, extra_features: Vec::new() }) + .expect("in-tree tool"); + + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + host, + "test", + "src/tools/rustfmt", + SourceType::InTree, + &[], + ); + + let dir = testdir(builder, compiler.host); + t!(fs::create_dir_all(&dir)); + cargo.env("RUSTFMT_TEST_DIR", dir); + + cargo.add_rustc_lib_path(builder, compiler); + + run_cargo_test(cargo, &[], &[], "rustfmt", "rustfmt", compiler, host, builder); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct RustDemangler { + stage: u32, + host: TargetSelection, +} + +impl Step for RustDemangler { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rust-demangler") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustDemangler { stage: run.builder.top_stage, host: run.target }); + } + + /// Runs `cargo test` for rust-demangler. + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let host = self.host; + let compiler = builder.compiler(stage, host); + + let rust_demangler = builder + .ensure(tool::RustDemangler { compiler, target: self.host, extra_features: Vec::new() }) + .expect("in-tree tool"); + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + host, + "test", + "src/tools/rust-demangler", + SourceType::InTree, + &[], + ); + + let dir = testdir(builder, compiler.host); + t!(fs::create_dir_all(&dir)); + + cargo.env("RUST_DEMANGLER_DRIVER_PATH", rust_demangler); + cargo.add_rustc_lib_path(builder, compiler); + + run_cargo_test( + cargo, + &[], + &[], + "rust-demangler", + "rust-demangler", + compiler, + host, + builder, + ); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Miri { + stage: u32, + host: TargetSelection, + target: TargetSelection, +} + +impl Miri { + /// Run `cargo miri setup` for the given target, return where the Miri sysroot was put. + pub fn build_miri_sysroot( + builder: &Builder<'_>, + compiler: Compiler, + miri: &Path, + target: TargetSelection, + ) -> String { + let miri_sysroot = builder.out.join(compiler.host.triple).join("miri-sysroot"); + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + compiler.host, + "run", + "src/tools/miri/cargo-miri", + SourceType::InTree, + &[], + ); + cargo.add_rustc_lib_path(builder, compiler); + cargo.arg("--").arg("miri").arg("setup"); + cargo.arg("--target").arg(target.rustc_target_arg()); + + // Tell `cargo miri setup` where to find the sources. + cargo.env("MIRI_LIB_SRC", builder.src.join("library")); + // Tell it where to find Miri. + cargo.env("MIRI", &miri); + // Tell it where to put the sysroot. + cargo.env("MIRI_SYSROOT", &miri_sysroot); + // Debug things. + cargo.env("RUST_BACKTRACE", "1"); + + let mut cargo = Command::from(cargo); + let _guard = builder.msg( + Kind::Build, + compiler.stage + 1, + "miri sysroot", + compiler.host, + compiler.host, + ); + builder.run(&mut cargo); + + // # Determine where Miri put its sysroot. + // To this end, we run `cargo miri setup --print-sysroot` and capture the output. + // (We do this separately from the above so that when the setup actually + // happens we get some output.) + // We re-use the `cargo` from above. + cargo.arg("--print-sysroot"); + + // FIXME: Is there a way in which we can re-use the usual `run` helpers? + if builder.config.dry_run() { + String::new() + } else { + builder.verbose(&format!("running: {cargo:?}")); + let out = + cargo.output().expect("We already ran `cargo miri setup` before and that worked"); + assert!(out.status.success(), "`cargo miri setup` returned with non-0 exit code"); + // Output is "\n". + let stdout = String::from_utf8(out.stdout) + .expect("`cargo miri setup` stdout is not valid UTF-8"); + let sysroot = stdout.trim_end(); + builder.verbose(&format!("`cargo miri setup --print-sysroot` said: {sysroot:?}")); + sysroot.to_owned() + } + } +} + +impl Step for Miri { + type Output = (); + const ONLY_HOSTS: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/miri") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Miri { + stage: run.builder.top_stage, + host: run.build_triple(), + target: run.target, + }); + } + + /// Runs `cargo test` for miri. + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let host = self.host; + let target = self.target; + let compiler = builder.compiler(stage, host); + // We need the stdlib for the *next* stage, as it was built with this compiler that also built Miri. + // Except if we are at stage 2, the bootstrap loop is complete and we can stick with our current stage. + let compiler_std = builder.compiler(if stage < 2 { stage + 1 } else { stage }, host); + + let miri = builder + .ensure(tool::Miri { compiler, target: self.host, extra_features: Vec::new() }) + .expect("in-tree tool"); + let _cargo_miri = builder + .ensure(tool::CargoMiri { compiler, target: self.host, extra_features: Vec::new() }) + .expect("in-tree tool"); + // The stdlib we need might be at a different stage. And just asking for the + // sysroot does not seem to populate it, so we do that first. + builder.ensure(compile::Std::new(compiler_std, host)); + let sysroot = builder.sysroot(compiler_std); + // We also need a Miri sysroot. + let miri_sysroot = Miri::build_miri_sysroot(builder, compiler, &miri, target); + + // # Run `cargo test`. + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + host, + "test", + "src/tools/miri", + SourceType::InTree, + &[], + ); + let _guard = builder.msg_sysroot_tool(Kind::Test, compiler.stage, "miri", host, target); + + cargo.add_rustc_lib_path(builder, compiler); + + // miri tests need to know about the stage sysroot + cargo.env("MIRI_SYSROOT", &miri_sysroot); + cargo.env("MIRI_HOST_SYSROOT", sysroot); + cargo.env("MIRI", &miri); + if builder.config.locked_deps { + // enforce lockfiles + cargo.env("CARGO_EXTRA_FLAGS", "--locked"); + } + + // Set the target. + cargo.env("MIRI_TEST_TARGET", target.rustc_target_arg()); + + // This can NOT be `run_cargo_test` since the Miri test runner + // does not understand the flags added by `add_flags_and_try_run_test`. + let mut cargo = prepare_cargo_test(cargo, &[], &[], "miri", compiler, target, builder); + { + let _time = helpers::timeit(&builder); + builder.run(&mut cargo); + } + + // Run it again for mir-opt-level 4 to catch some miscompilations. + if builder.config.test_args().is_empty() { + cargo.env("MIRIFLAGS", "-O -Zmir-opt-level=4 -Cdebug-assertions=yes"); + // Optimizations can change backtraces + cargo.env("MIRI_SKIP_UI_CHECKS", "1"); + // `MIRI_SKIP_UI_CHECKS` and `RUSTC_BLESS` are incompatible + cargo.env_remove("RUSTC_BLESS"); + // Optimizations can change error locations and remove UB so don't run `fail` tests. + cargo.args(&["tests/pass", "tests/panic"]); + + let mut cargo = prepare_cargo_test(cargo, &[], &[], "miri", compiler, target, builder); + { + let _time = helpers::timeit(&builder); + builder.run(&mut cargo); + } + } + + // # Run `cargo miri test`. + // This is just a smoke test (Miri's own CI invokes this in a bunch of different ways and ensures + // that we get the desired output), but that is sufficient to make sure that the libtest harness + // itself executes properly under Miri. + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + host, + "run", + "src/tools/miri/cargo-miri", + SourceType::Submodule, + &[], + ); + cargo.add_rustc_lib_path(builder, compiler); + cargo.arg("--").arg("miri").arg("test"); + if builder.config.locked_deps { + cargo.arg("--locked"); + } + cargo + .arg("--manifest-path") + .arg(builder.src.join("src/tools/miri/test-cargo-miri/Cargo.toml")); + cargo.arg("--target").arg(target.rustc_target_arg()); + cargo.arg("--tests"); // don't run doctests, they are too confused by the staging + cargo.arg("--").args(builder.config.test_args()); + + // Tell `cargo miri` where to find things. + cargo.env("MIRI_SYSROOT", &miri_sysroot); + cargo.env("MIRI_HOST_SYSROOT", sysroot); + cargo.env("MIRI", &miri); + // Debug things. + cargo.env("RUST_BACKTRACE", "1"); + + let mut cargo = Command::from(cargo); + { + let _time = helpers::timeit(&builder); + builder.run(&mut cargo); + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct CompiletestTest { + host: TargetSelection, +} + +impl Step for CompiletestTest { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/compiletest") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(CompiletestTest { host: run.target }); + } + + /// Runs `cargo test` for compiletest. + fn run(self, builder: &Builder<'_>) { + let host = self.host; + let compiler = builder.compiler(builder.top_stage, host); + + // We need `ToolStd` for the locally-built sysroot because + // compiletest uses unstable features of the `test` crate. + builder.ensure(compile::Std::new(compiler, host)); + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolStd, + host, + "test", + "src/tools/compiletest", + SourceType::InTree, + &[], + ); + cargo.allow_features("test"); + run_cargo_test( + cargo, + &[], + &[], + "compiletest", + "compiletest self test", + compiler, + host, + builder, + ); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Clippy { + stage: u32, + host: TargetSelection, +} + +impl Step for Clippy { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/clippy") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Clippy { stage: run.builder.top_stage, host: run.target }); + } + + /// Runs `cargo test` for clippy. + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let host = self.host; + let compiler = builder.compiler(stage, host); + + builder + .ensure(tool::Clippy { compiler, target: self.host, extra_features: Vec::new() }) + .expect("in-tree tool"); + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + host, + "test", + "src/tools/clippy", + SourceType::InTree, + &[], + ); + + cargo.env("RUSTC_TEST_SUITE", builder.rustc(compiler)); + cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler)); + let host_libs = builder.stage_out(compiler, Mode::ToolRustc).join(builder.cargo_dir()); + cargo.env("HOST_LIBS", host_libs); + + cargo.add_rustc_lib_path(builder, compiler); + let mut cargo = prepare_cargo_test(cargo, &[], &[], "clippy", compiler, host, builder); + + let _guard = builder.msg_sysroot_tool(Kind::Test, compiler.stage, "clippy", host, host); + + // Clippy reports errors if it blessed the outputs + if builder.run_cmd(BootstrapCommand::from(&mut cargo).allow_failure()) { + // The tests succeeded; nothing to do. + return; + } + + if !builder.config.cmd.bless() { + crate::exit!(1); + } + } +} + +fn path_for_cargo(builder: &Builder<'_>, compiler: Compiler) -> OsString { + // Configure PATH to find the right rustc. NB. we have to use PATH + // and not RUSTC because the Cargo test suite has tests that will + // fail if rustc is not spelled `rustc`. + let path = builder.sysroot(compiler).join("bin"); + let old_path = env::var_os("PATH").unwrap_or_default(); + env::join_paths(iter::once(path).chain(env::split_paths(&old_path))).expect("") +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustdocTheme { + pub compiler: Compiler, +} + +impl Step for RustdocTheme { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rustdoc-themes") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + + run.builder.ensure(RustdocTheme { compiler }); + } + + fn run(self, builder: &Builder<'_>) { + let rustdoc = builder.bootstrap_out.join("rustdoc"); + let mut cmd = builder.tool_cmd(Tool::RustdocTheme); + cmd.arg(rustdoc.to_str().unwrap()) + .arg(builder.src.join("src/librustdoc/html/static/css/rustdoc.css").to_str().unwrap()) + .env("RUSTC_STAGE", self.compiler.stage.to_string()) + .env("RUSTC_SYSROOT", builder.sysroot(self.compiler)) + .env("RUSTDOC_LIBDIR", builder.sysroot_libdir(self.compiler, self.compiler.host)) + .env("CFG_RELEASE_CHANNEL", &builder.config.channel) + .env("RUSTDOC_REAL", builder.rustdoc(self.compiler)) + .env("RUSTC_BOOTSTRAP", "1"); + if let Some(linker) = builder.linker(self.compiler.host) { + cmd.env("RUSTDOC_LINKER", linker); + } + if builder.is_fuse_ld_lld(self.compiler.host) { + cmd.env( + "RUSTDOC_LLD_NO_THREADS", + helpers::lld_flag_no_threads(self.compiler.host.contains("windows")), + ); + } + builder.run_delaying_failure(&mut cmd); + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustdocJSStd { + pub target: TargetSelection, +} + +impl Step for RustdocJSStd { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = run.builder.config.nodejs.is_some(); + run.suite_path("tests/rustdoc-js-std").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustdocJSStd { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let nodejs = + builder.config.nodejs.as_ref().expect("need nodejs to run rustdoc-js-std tests"); + let mut command = Command::new(nodejs); + command + .arg(builder.src.join("src/tools/rustdoc-js/tester.js")) + .arg("--crate-name") + .arg("std") + .arg("--resource-suffix") + .arg(&builder.version) + .arg("--doc-folder") + .arg(builder.doc_out(self.target)) + .arg("--test-folder") + .arg(builder.src.join("tests/rustdoc-js-std")); + for path in &builder.paths { + if let Some(p) = helpers::is_valid_test_suite_arg(path, "tests/rustdoc-js-std", builder) + { + if !p.ends_with(".js") { + eprintln!("A non-js file was given: `{}`", path.display()); + panic!("Cannot run rustdoc-js-std tests"); + } + command.arg("--test-file").arg(path); + } + } + builder.ensure(crate::core::build_steps::doc::Std::new( + builder.top_stage, + self.target, + builder, + DocumentationFormat::HTML, + )); + let _guard = builder.msg( + Kind::Test, + builder.top_stage, + "rustdoc-js-std", + builder.config.build, + self.target, + ); + builder.run(&mut command); + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustdocJSNotStd { + pub target: TargetSelection, + pub compiler: Compiler, +} + +impl Step for RustdocJSNotStd { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = run.builder.config.nodejs.is_some(); + run.suite_path("tests/rustdoc-js").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + run.builder.ensure(RustdocJSNotStd { target: run.target, compiler }); + } + + fn run(self, builder: &Builder<'_>) { + builder.ensure(Compiletest { + compiler: self.compiler, + target: self.target, + mode: "js-doc-test", + suite: "rustdoc-js", + path: "tests/rustdoc-js", + compare_mode: None, + }); + } +} + +fn get_browser_ui_test_version_inner(npm: &Path, global: bool) -> Option { + let mut command = Command::new(&npm); + command.arg("list").arg("--parseable").arg("--long").arg("--depth=0"); + if global { + command.arg("--global"); + } + let lines = command + .output() + .map(|output| String::from_utf8_lossy(&output.stdout).into_owned()) + .unwrap_or(String::new()); + lines + .lines() + .find_map(|l| l.split(':').nth(1)?.strip_prefix("browser-ui-test@")) + .map(|v| v.to_owned()) +} + +fn get_browser_ui_test_version(npm: &Path) -> Option { + get_browser_ui_test_version_inner(npm, false) + .or_else(|| get_browser_ui_test_version_inner(npm, true)) +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustdocGUI { + pub target: TargetSelection, + pub compiler: Compiler, +} + +impl Step for RustdocGUI { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + let run = run.suite_path("tests/rustdoc-gui"); + run.lazy_default_condition(Box::new(move || { + builder.config.nodejs.is_some() + && builder + .config + .npm + .as_ref() + .map(|p| get_browser_ui_test_version(p).is_some()) + .unwrap_or(false) + })) + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + run.builder.ensure(RustdocGUI { target: run.target, compiler }); + } + + fn run(self, builder: &Builder<'_>) { + builder.ensure(compile::Std::new(self.compiler, self.target)); + + let mut cmd = builder.tool_cmd(Tool::RustdocGUITest); + + let out_dir = builder.test_out(self.target).join("rustdoc-gui"); + builder.clear_if_dirty(&out_dir, &builder.rustdoc(self.compiler)); + + if let Some(src) = builder.config.src.to_str() { + cmd.arg("--rust-src").arg(src); + } + + if let Some(out_dir) = out_dir.to_str() { + cmd.arg("--out-dir").arg(out_dir); + } + + if let Some(initial_cargo) = builder.config.initial_cargo.to_str() { + cmd.arg("--initial-cargo").arg(initial_cargo); + } + + cmd.arg("--jobs").arg(builder.jobs().to_string()); + + cmd.env("RUSTDOC", builder.rustdoc(self.compiler)) + .env("RUSTC", builder.rustc(self.compiler)); + + for path in &builder.paths { + if let Some(p) = helpers::is_valid_test_suite_arg(path, "tests/rustdoc-gui", builder) { + if !p.ends_with(".goml") { + eprintln!("A non-goml file was given: `{}`", path.display()); + panic!("Cannot run rustdoc-gui tests"); + } + if let Some(name) = path.file_name().and_then(|f| f.to_str()) { + cmd.arg("--goml-file").arg(name); + } + } + } + + for test_arg in builder.config.test_args() { + cmd.arg("--test-arg").arg(test_arg); + } + + if let Some(ref nodejs) = builder.config.nodejs { + cmd.arg("--nodejs").arg(nodejs); + } + + if let Some(ref npm) = builder.config.npm { + cmd.arg("--npm").arg(npm); + } + + let _time = helpers::timeit(&builder); + let _guard = builder.msg_sysroot_tool( + Kind::Test, + self.compiler.stage, + "rustdoc-gui", + self.compiler.host, + self.target, + ); + try_run_tests(builder, &mut cmd, true); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Tidy; + +impl Step for Tidy { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + /// Runs the `tidy` tool. + /// + /// This tool in `src/tools` checks up on various bits and pieces of style and + /// otherwise just implements a few lint-like checks that are specific to the + /// compiler itself. + /// + /// Once tidy passes, this step also runs `fmt --check` if tests are being run + /// for the `dev` or `nightly` channels. + fn run(self, builder: &Builder<'_>) { + let mut cmd = builder.tool_cmd(Tool::Tidy); + cmd.arg(&builder.src); + cmd.arg(&builder.initial_cargo); + cmd.arg(&builder.out); + // Tidy is heavily IO constrained. Still respect `-j`, but use a higher limit if `jobs` hasn't been configured. + let jobs = builder.config.jobs.unwrap_or_else(|| { + 8 * std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32 + }); + cmd.arg(jobs.to_string()); + if builder.is_verbose() { + cmd.arg("--verbose"); + } + if builder.config.cmd.bless() { + cmd.arg("--bless"); + } + if let Some(s) = builder.config.cmd.extra_checks() { + cmd.arg(format!("--extra-checks={s}")); + } + let mut args = std::env::args_os(); + if let Some(_) = args.find(|arg| arg == OsStr::new("--")) { + cmd.arg("--"); + cmd.args(args); + } + + if builder.config.channel == "dev" || builder.config.channel == "nightly" { + builder.info("fmt check"); + if builder.initial_rustfmt().is_none() { + let inferred_rustfmt_dir = builder.initial_rustc.parent().unwrap(); + eprintln!( + "\ +ERROR: no `rustfmt` binary found in {PATH} +INFO: `rust.channel` is currently set to \"{CHAN}\" +HELP: if you are testing a beta branch, set `rust.channel` to \"beta\" in the `config.toml` file +HELP: to skip test's attempt to check tidiness, pass `--skip src/tools/tidy` to `x.py test`", + PATH = inferred_rustfmt_dir.display(), + CHAN = builder.config.channel, + ); + crate::exit!(1); + } + crate::core::build_steps::format::format(&builder, !builder.config.cmd.bless(), &[]); + } + + builder.info("tidy check"); + builder.run_delaying_failure(&mut cmd); + + builder.ensure(ExpandYamlAnchors); + + builder.info("x.py completions check"); + let [bash, zsh, fish, powershell] = ["x.py.sh", "x.py.zsh", "x.py.fish", "x.py.ps1"] + .map(|filename| builder.src.join("src/etc/completions").join(filename)); + if builder.config.cmd.bless() { + builder.ensure(crate::core::build_steps::run::GenerateCompletions); + } else if get_completion(shells::Bash, &bash).is_some() + || get_completion(shells::Fish, &fish).is_some() + || get_completion(shells::PowerShell, &powershell).is_some() + || crate::flags::get_completion(shells::Zsh, &zsh).is_some() + { + eprintln!( + "x.py completions were changed; run `x.py run generate-completions` to update them" + ); + crate::exit!(1); + } + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/tidy") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Tidy); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct ExpandYamlAnchors; + +impl Step for ExpandYamlAnchors { + type Output = (); + const ONLY_HOSTS: bool = true; + + /// Ensure the `generate-ci-config` tool was run locally. + /// + /// The tool in `src/tools` reads the CI definition in `src/ci/builders.yml` and generates the + /// appropriate configuration for all our CI providers. This step ensures the tool was called + /// by the user before committing CI changes. + fn run(self, builder: &Builder<'_>) { + // NOTE: `.github/` is not included in dist-src tarballs + if !builder.src.join(".github/workflows/ci.yml").exists() { + builder.info("Skipping YAML anchors check: GitHub Actions config not found"); + return; + } + builder.info("Ensuring the YAML anchors in the GitHub Actions config were expanded"); + builder.run_delaying_failure( + &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("check").arg(&builder.src), + ); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/expand-yaml-anchors") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(ExpandYamlAnchors); + } +} + +fn testdir(builder: &Builder<'_>, host: TargetSelection) -> PathBuf { + builder.out.join(host.triple).join("test") +} + +macro_rules! default_test { + ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr }) => { + test!($name { path: $path, mode: $mode, suite: $suite, default: true, host: false }); + }; +} + +macro_rules! default_test_with_compare_mode { + ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, + compare_mode: $compare_mode:expr }) => { + test_with_compare_mode!($name { + path: $path, + mode: $mode, + suite: $suite, + default: true, + host: false, + compare_mode: $compare_mode + }); + }; +} + +macro_rules! host_test { + ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr }) => { + test!($name { path: $path, mode: $mode, suite: $suite, default: true, host: true }); + }; +} + +macro_rules! test { + ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, default: $default:expr, + host: $host:expr }) => { + test_definitions!($name { + path: $path, + mode: $mode, + suite: $suite, + default: $default, + host: $host, + compare_mode: None + }); + }; +} + +macro_rules! test_with_compare_mode { + ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, default: $default:expr, + host: $host:expr, compare_mode: $compare_mode:expr }) => { + test_definitions!($name { + path: $path, + mode: $mode, + suite: $suite, + default: $default, + host: $host, + compare_mode: Some($compare_mode) + }); + }; +} + +macro_rules! test_definitions { + ($name:ident { + path: $path:expr, + mode: $mode:expr, + suite: $suite:expr, + default: $default:expr, + host: $host:expr, + compare_mode: $compare_mode:expr + }) => { + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] + pub struct $name { + pub compiler: Compiler, + pub target: TargetSelection, + } + + impl Step for $name { + type Output = (); + const DEFAULT: bool = $default; + const ONLY_HOSTS: bool = $host; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.suite_path($path) + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + + run.builder.ensure($name { compiler, target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + builder.ensure(Compiletest { + compiler: self.compiler, + target: self.target, + mode: $mode, + suite: $suite, + path: $path, + compare_mode: $compare_mode, + }) + } + } + }; +} + +/// Declares an alias for running the [`Coverage`] tests in only one mode. +/// Adapted from [`test_definitions`]. +macro_rules! coverage_test_alias { + ($name:ident { + alias_and_mode: $alias_and_mode:expr, + default: $default:expr, + only_hosts: $only_hosts:expr $(,)? + }) => { + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] + pub struct $name { + pub compiler: Compiler, + pub target: TargetSelection, + } + + impl $name { + const MODE: &'static str = $alias_and_mode; + } + + impl Step for $name { + type Output = (); + const DEFAULT: bool = $default; + const ONLY_HOSTS: bool = $only_hosts; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias($alias_and_mode) + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + + run.builder.ensure($name { compiler, target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + Coverage { compiler: self.compiler, target: self.target } + .run_unified_suite(builder, Self::MODE) + } + } + }; +} + +default_test!(Ui { path: "tests/ui", mode: "ui", suite: "ui" }); + +default_test!(RunPassValgrind { + path: "tests/run-pass-valgrind", + mode: "run-pass-valgrind", + suite: "run-pass-valgrind" +}); + +default_test!(Codegen { path: "tests/codegen", mode: "codegen", suite: "codegen" }); + +default_test!(CodegenUnits { + path: "tests/codegen-units", + mode: "codegen-units", + suite: "codegen-units" +}); + +default_test!(Incremental { path: "tests/incremental", mode: "incremental", suite: "incremental" }); + +default_test_with_compare_mode!(Debuginfo { + path: "tests/debuginfo", + mode: "debuginfo", + suite: "debuginfo", + compare_mode: "split-dwarf" +}); + +host_test!(UiFullDeps { path: "tests/ui-fulldeps", mode: "ui", suite: "ui-fulldeps" }); + +host_test!(Rustdoc { path: "tests/rustdoc", mode: "rustdoc", suite: "rustdoc" }); +host_test!(RustdocUi { path: "tests/rustdoc-ui", mode: "ui", suite: "rustdoc-ui" }); + +host_test!(RustdocJson { path: "tests/rustdoc-json", mode: "rustdoc-json", suite: "rustdoc-json" }); + +host_test!(Pretty { path: "tests/pretty", mode: "pretty", suite: "pretty" }); + +default_test!(RunMake { path: "tests/run-make", mode: "run-make", suite: "run-make" }); + +host_test!(RunMakeFullDeps { + path: "tests/run-make-fulldeps", + mode: "run-make", + suite: "run-make-fulldeps" +}); + +default_test!(Assembly { path: "tests/assembly", mode: "assembly", suite: "assembly" }); + +/// Custom test step that is responsible for running the coverage tests +/// in multiple different modes. +/// +/// Each individual mode also has its own alias that will run the tests in +/// just that mode. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Coverage { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Coverage { + const PATH: &'static str = "tests/coverage"; + const SUITE: &'static str = "coverage"; + + fn run_unified_suite(&self, builder: &Builder<'_>, mode: &'static str) { + builder.ensure(Compiletest { + compiler: self.compiler, + target: self.target, + mode, + suite: Self::SUITE, + path: Self::PATH, + compare_mode: None, + }) + } +} + +impl Step for Coverage { + type Output = (); + const DEFAULT: bool = false; + const ONLY_HOSTS: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.suite_path(Self::PATH) + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + + run.builder.ensure(Coverage { compiler, target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + self.run_unified_suite(builder, CoverageMap::MODE); + self.run_unified_suite(builder, CoverageRun::MODE); + } +} + +// Aliases for running the coverage tests in only one mode. +coverage_test_alias!(CoverageMap { + alias_and_mode: "coverage-map", + default: true, + only_hosts: false, +}); +coverage_test_alias!(CoverageRun { + alias_and_mode: "coverage-run", + default: true, + only_hosts: true, +}); + +host_test!(CoverageRunRustdoc { + path: "tests/coverage-run-rustdoc", + mode: "coverage-run", + suite: "coverage-run-rustdoc" +}); + +// For the mir-opt suite we do not use macros, as we need custom behavior when blessing. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct MirOpt { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for MirOpt { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.suite_path("tests/mir-opt") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + run.builder.ensure(MirOpt { compiler, target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let run = |target| { + builder.ensure(Compiletest { + compiler: self.compiler, + target, + mode: "mir-opt", + suite: "mir-opt", + path: "tests/mir-opt", + compare_mode: None, + }) + }; + + // We use custom logic to bless the mir-opt suite: mir-opt tests have multiple variants + // (32bit vs 64bit, and panic=abort vs panic=unwind), and all of them needs to be blessed. + // When blessing, we try best-effort to also bless the other variants, to aid developers. + if builder.config.cmd.bless() { + let targets = MIR_OPT_BLESS_TARGET_MAPPING + .iter() + .filter(|(target_32bit, target_64bit)| { + *target_32bit == &*self.target.triple || *target_64bit == &*self.target.triple + }) + .next() + .map(|(target_32bit, target_64bit)| { + let target_32bit = TargetSelection::from_user(target_32bit); + let target_64bit = TargetSelection::from_user(target_64bit); + + // Running compiletest requires a C compiler to be available, but it might not + // have been detected by bootstrap if the target we're testing wasn't in the + // --target flags. + if !builder.cc.borrow().contains_key(&target_32bit) { + utils::cc_detect::find_target(builder, target_32bit); + } + if !builder.cc.borrow().contains_key(&target_64bit) { + utils::cc_detect::find_target(builder, target_64bit); + } + + vec![target_32bit, target_64bit] + }) + .unwrap_or_else(|| { + eprintln!( + "\ +Note that not all variants of mir-opt tests are going to be blessed, as no mapping between +a 32bit and a 64bit target was found for {target}. +You can add that mapping by changing MIR_OPT_BLESS_TARGET_MAPPING in src/bootstrap/test.rs", + target = self.target, + ); + vec![self.target] + }); + + for target in targets { + run(target); + + let panic_abort_target = builder.ensure(MirOptPanicAbortSyntheticTarget { + compiler: self.compiler, + base: target, + }); + run(panic_abort_target); + } + } else { + run(self.target); + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +struct Compiletest { + compiler: Compiler, + target: TargetSelection, + mode: &'static str, + suite: &'static str, + path: &'static str, + compare_mode: Option<&'static str>, +} + +impl Step for Compiletest { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Executes the `compiletest` tool to run a suite of tests. + /// + /// Compiles all tests with `compiler` for `target` with the specified + /// compiletest `mode` and `suite` arguments. For example `mode` can be + /// "run-pass" or `suite` can be something like `debuginfo`. + fn run(self, builder: &Builder<'_>) { + if builder.top_stage == 0 && env::var("COMPILETEST_FORCE_STAGE0").is_err() { + eprintln!("\ +ERROR: `--stage 0` runs compiletest on the beta compiler, not your local changes, and will almost always cause tests to fail +HELP: to test the compiler, use `--stage 1` instead +HELP: to test the standard library, use `--stage 0 library/std` instead +NOTE: if you're sure you want to do this, please open an issue as to why. In the meantime, you can override this with `COMPILETEST_FORCE_STAGE0=1`." + ); + crate::exit!(1); + } + + let mut compiler = self.compiler; + let target = self.target; + let mode = self.mode; + let suite = self.suite; + + // Path for test suite + let suite_path = self.path; + + // Skip codegen tests if they aren't enabled in configuration. + if !builder.config.codegen_tests && suite == "codegen" { + return; + } + + // Support stage 1 ui-fulldeps. This is somewhat complicated: ui-fulldeps tests for the most + // part test the *API* of the compiler, not how it compiles a given file. As a result, we + // can run them against the stage 1 sources as long as we build them with the stage 0 + // bootstrap compiler. + // NOTE: Only stage 1 is special cased because we need the rustc_private artifacts to match the + // running compiler in stage 2 when plugins run. + let stage_id = if suite == "ui-fulldeps" && compiler.stage == 1 { + compiler = builder.compiler(compiler.stage - 1, target); + format!("stage{}-{}", compiler.stage + 1, target) + } else { + format!("stage{}-{}", compiler.stage, target) + }; + + if suite.ends_with("fulldeps") { + builder.ensure(compile::Rustc::new(compiler, target)); + } + + if suite == "debuginfo" { + builder + .ensure(dist::DebuggerScripts { sysroot: builder.sysroot(compiler), host: target }); + } + + builder.ensure(compile::Std::new(compiler, target)); + // ensure that `libproc_macro` is available on the host. + builder.ensure(compile::Std::new(compiler, compiler.host)); + + // Also provide `rust_test_helpers` for the host. + builder.ensure(TestHelpers { target: compiler.host }); + + // As well as the target, except for plain wasm32, which can't build it + if !target.contains("wasm") || target.contains("emscripten") { + builder.ensure(TestHelpers { target }); + } + + builder.ensure(RemoteCopyLibs { compiler, target }); + + let mut cmd = builder.tool_cmd(Tool::Compiletest); + + // compiletest currently has... a lot of arguments, so let's just pass all + // of them! + + cmd.arg("--compile-lib-path").arg(builder.rustc_libdir(compiler)); + cmd.arg("--run-lib-path").arg(builder.sysroot_libdir(compiler, target)); + cmd.arg("--rustc-path").arg(builder.rustc(compiler)); + + let is_rustdoc = suite.ends_with("rustdoc-ui") || suite.ends_with("rustdoc-js"); + + // Avoid depending on rustdoc when we don't need it. + if mode == "rustdoc" + || mode == "run-make" + || (mode == "ui" && is_rustdoc) + || mode == "js-doc-test" + || mode == "rustdoc-json" + || suite == "coverage-run-rustdoc" + { + cmd.arg("--rustdoc-path").arg(builder.rustdoc(compiler)); + } + + if mode == "rustdoc-json" { + // Use the beta compiler for jsondocck + let json_compiler = compiler.with_stage(0); + cmd.arg("--jsondocck-path") + .arg(builder.ensure(tool::JsonDocCk { compiler: json_compiler, target })); + cmd.arg("--jsondoclint-path") + .arg(builder.ensure(tool::JsonDocLint { compiler: json_compiler, target })); + } + + if mode == "coverage-map" { + let coverage_dump = builder.ensure(tool::CoverageDump { + compiler: compiler.with_stage(0), + target: compiler.host, + }); + cmd.arg("--coverage-dump-path").arg(coverage_dump); + } + + if mode == "coverage-run" { + // The demangler doesn't need the current compiler, so we can avoid + // unnecessary rebuilds by using the bootstrap compiler instead. + let rust_demangler = builder + .ensure(tool::RustDemangler { + compiler: compiler.with_stage(0), + target: compiler.host, + extra_features: Vec::new(), + }) + .expect("in-tree tool"); + cmd.arg("--rust-demangler-path").arg(rust_demangler); + } + + cmd.arg("--src-base").arg(builder.src.join("tests").join(suite)); + cmd.arg("--build-base").arg(testdir(builder, compiler.host).join(suite)); + + // When top stage is 0, that means that we're testing an externally provided compiler. + // In that case we need to use its specific sysroot for tests to pass. + let sysroot = if builder.top_stage == 0 { + builder.initial_sysroot.clone() + } else { + builder.sysroot(compiler).to_path_buf() + }; + cmd.arg("--sysroot-base").arg(sysroot); + cmd.arg("--stage-id").arg(stage_id); + cmd.arg("--suite").arg(suite); + cmd.arg("--mode").arg(mode); + cmd.arg("--target").arg(target.rustc_target_arg()); + cmd.arg("--host").arg(&*compiler.host.triple); + cmd.arg("--llvm-filecheck").arg(builder.llvm_filecheck(builder.config.build)); + + if builder.config.cmd.bless() { + cmd.arg("--bless"); + } + + if builder.config.cmd.force_rerun() { + cmd.arg("--force-rerun"); + } + + let compare_mode = + builder.config.cmd.compare_mode().or_else(|| { + if builder.config.test_compare_mode { self.compare_mode } else { None } + }); + + if let Some(ref pass) = builder.config.cmd.pass() { + cmd.arg("--pass"); + cmd.arg(pass); + } + + if let Some(ref run) = builder.config.cmd.run() { + cmd.arg("--run"); + cmd.arg(run); + } + + if let Some(ref nodejs) = builder.config.nodejs { + cmd.arg("--nodejs").arg(nodejs); + } else if mode == "js-doc-test" { + panic!("need nodejs to run js-doc-test suite"); + } + if let Some(ref npm) = builder.config.npm { + cmd.arg("--npm").arg(npm); + } + if builder.config.rust_optimize_tests { + cmd.arg("--optimize-tests"); + } + if builder.config.cmd.only_modified() { + cmd.arg("--only-modified"); + } + + let mut flags = if is_rustdoc { Vec::new() } else { vec!["-Crpath".to_string()] }; + flags.push(format!("-Cdebuginfo={}", builder.config.rust_debuginfo_level_tests)); + flags.extend(builder.config.cmd.rustc_args().iter().map(|s| s.to_string())); + + if let Some(linker) = builder.linker(target) { + cmd.arg("--target-linker").arg(linker); + } + if let Some(linker) = builder.linker(compiler.host) { + cmd.arg("--host-linker").arg(linker); + } + + let mut hostflags = flags.clone(); + hostflags.push(format!("-Lnative={}", builder.test_helpers_out(compiler.host).display())); + hostflags.extend(builder.lld_flags(compiler.host)); + for flag in hostflags { + cmd.arg("--host-rustcflags").arg(flag); + } + + let mut targetflags = flags; + targetflags.push(format!("-Lnative={}", builder.test_helpers_out(target).display())); + targetflags.extend(builder.lld_flags(target)); + for flag in targetflags { + cmd.arg("--target-rustcflags").arg(flag); + } + + cmd.arg("--python").arg(builder.python()); + + if let Some(ref gdb) = builder.config.gdb { + cmd.arg("--gdb").arg(gdb); + } + + let run = |cmd: &mut Command| { + cmd.output().map(|output| { + String::from_utf8_lossy(&output.stdout) + .lines() + .next() + .unwrap_or_else(|| panic!("{:?} failed {:?}", cmd, output)) + .to_string() + }) + }; + let lldb_exe = "lldb"; + let lldb_version = Command::new(lldb_exe) + .arg("--version") + .output() + .map(|output| String::from_utf8_lossy(&output.stdout).to_string()) + .ok(); + if let Some(ref vers) = lldb_version { + cmd.arg("--lldb-version").arg(vers); + let lldb_python_dir = run(Command::new(lldb_exe).arg("-P")).ok(); + if let Some(ref dir) = lldb_python_dir { + cmd.arg("--lldb-python-dir").arg(dir); + } + } + + if helpers::forcing_clang_based_tests() { + let clang_exe = builder.llvm_out(target).join("bin").join("clang"); + cmd.arg("--run-clang-based-tests-with").arg(clang_exe); + } + + for exclude in &builder.config.skip { + cmd.arg("--skip"); + cmd.arg(&exclude); + } + + // Get paths from cmd args + let paths = match &builder.config.cmd { + Subcommand::Test { .. } => &builder.config.paths[..], + _ => &[], + }; + + // Get test-args by striping suite path + let mut test_args: Vec<&str> = paths + .iter() + .filter_map(|p| helpers::is_valid_test_suite_arg(p, suite_path, builder)) + .collect(); + + test_args.append(&mut builder.config.test_args()); + + // On Windows, replace forward slashes in test-args by backslashes + // so the correct filters are passed to libtest + if cfg!(windows) { + let test_args_win: Vec = + test_args.iter().map(|s| s.replace("/", "\\")).collect(); + cmd.args(&test_args_win); + } else { + cmd.args(&test_args); + } + + if builder.is_verbose() { + cmd.arg("--verbose"); + } + + cmd.arg("--json"); + + let mut llvm_components_passed = false; + let mut copts_passed = false; + if builder.config.llvm_enabled() { + let llvm::LlvmResult { llvm_config, .. } = + builder.ensure(llvm::Llvm { target: builder.config.build }); + if !builder.config.dry_run() { + let llvm_version = output(Command::new(&llvm_config).arg("--version")); + let llvm_components = output(Command::new(&llvm_config).arg("--components")); + // Remove trailing newline from llvm-config output. + cmd.arg("--llvm-version") + .arg(llvm_version.trim()) + .arg("--llvm-components") + .arg(llvm_components.trim()); + llvm_components_passed = true; + } + if !builder.is_rust_llvm(target) { + cmd.arg("--system-llvm"); + } + + // Tests that use compiler libraries may inherit the `-lLLVM` link + // requirement, but the `-L` library path is not propagated across + // separate compilations. We can add LLVM's library path to the + // platform-specific environment variable as a workaround. + if !builder.config.dry_run() && suite.ends_with("fulldeps") { + let llvm_libdir = output(Command::new(&llvm_config).arg("--libdir")); + add_link_lib_path(vec![llvm_libdir.trim().into()], &mut cmd); + } + + if !builder.config.dry_run() + && (matches!(suite, "run-make" | "run-make-fulldeps") || mode == "coverage-run") + { + // The llvm/bin directory contains many useful cross-platform + // tools. Pass the path to run-make tests so they can use them. + // (The coverage-run tests also need these tools to process + // coverage reports.) + let llvm_bin_path = llvm_config + .parent() + .expect("Expected llvm-config to be contained in directory"); + assert!(llvm_bin_path.is_dir()); + cmd.arg("--llvm-bin-dir").arg(llvm_bin_path); + } + + if !builder.config.dry_run() && matches!(suite, "run-make" | "run-make-fulldeps") { + // If LLD is available, add it to the PATH + if builder.config.lld_enabled { + let lld_install_root = + builder.ensure(llvm::Lld { target: builder.config.build }); + + let lld_bin_path = lld_install_root.join("bin"); + + let old_path = env::var_os("PATH").unwrap_or_default(); + let new_path = env::join_paths( + std::iter::once(lld_bin_path).chain(env::split_paths(&old_path)), + ) + .expect("Could not add LLD bin path to PATH"); + cmd.env("PATH", new_path); + } + } + } + + // Only pass correct values for these flags for the `run-make` suite as it + // requires that a C++ compiler was configured which isn't always the case. + if !builder.config.dry_run() && matches!(suite, "run-make" | "run-make-fulldeps") { + cmd.arg("--cc") + .arg(builder.cc(target)) + .arg("--cxx") + .arg(builder.cxx(target).unwrap()) + .arg("--cflags") + .arg(builder.cflags(target, GitRepo::Rustc, CLang::C).join(" ")) + .arg("--cxxflags") + .arg(builder.cflags(target, GitRepo::Rustc, CLang::Cxx).join(" ")); + copts_passed = true; + if let Some(ar) = builder.ar(target) { + cmd.arg("--ar").arg(ar); + } + } + + if !llvm_components_passed { + cmd.arg("--llvm-components").arg(""); + } + if !copts_passed { + cmd.arg("--cc") + .arg("") + .arg("--cxx") + .arg("") + .arg("--cflags") + .arg("") + .arg("--cxxflags") + .arg(""); + } + + if builder.remote_tested(target) { + cmd.arg("--remote-test-client").arg(builder.tool_exe(Tool::RemoteTestClient)); + } + + // Running a C compiler on MSVC requires a few env vars to be set, to be + // sure to set them here. + // + // Note that if we encounter `PATH` we make sure to append to our own `PATH` + // rather than stomp over it. + if !builder.config.dry_run() && target.contains("msvc") { + for &(ref k, ref v) in builder.cc.borrow()[&target].env() { + if k != "PATH" { + cmd.env(k, v); + } + } + } + cmd.env("RUSTC_BOOTSTRAP", "1"); + // Override the rustc version used in symbol hashes to reduce the amount of normalization + // needed when diffing test output. + cmd.env("RUSTC_FORCE_RUSTC_VERSION", "compiletest"); + cmd.env("DOC_RUST_LANG_ORG_CHANNEL", builder.doc_rust_lang_org_channel()); + builder.add_rust_test_threads(&mut cmd); + + if builder.config.sanitizers_enabled(target) { + cmd.env("RUSTC_SANITIZER_SUPPORT", "1"); + } + + if builder.config.profiler_enabled(target) { + cmd.env("RUSTC_PROFILER_SUPPORT", "1"); + } + + cmd.env("RUST_TEST_TMPDIR", builder.tempdir()); + + cmd.arg("--adb-path").arg("adb"); + cmd.arg("--adb-test-dir").arg(ADB_TEST_DIR); + if target.contains("android") && !builder.config.dry_run() { + // Assume that cc for this target comes from the android sysroot + cmd.arg("--android-cross-path") + .arg(builder.cc(target).parent().unwrap().parent().unwrap()); + } else { + cmd.arg("--android-cross-path").arg(""); + } + + if builder.config.cmd.rustfix_coverage() { + cmd.arg("--rustfix-coverage"); + } + + cmd.env("BOOTSTRAP_CARGO", &builder.initial_cargo); + + cmd.arg("--channel").arg(&builder.config.channel); + + if !builder.config.omit_git_hash { + cmd.arg("--git-hash"); + } + + let git_config = builder.config.git_config(); + cmd.arg("--git-repository").arg(git_config.git_repository); + cmd.arg("--nightly-branch").arg(git_config.nightly_branch); + + builder.ci_env.force_coloring_in_ci(&mut cmd); + + #[cfg(feature = "build-metrics")] + builder.metrics.begin_test_suite( + build_helper::metrics::TestSuiteMetadata::Compiletest { + suite: suite.into(), + mode: mode.into(), + compare_mode: None, + target: self.target.triple.to_string(), + host: self.compiler.host.triple.to_string(), + stage: self.compiler.stage, + }, + builder, + ); + + let _group = builder.msg( + Kind::Test, + compiler.stage, + &format!("compiletest suite={suite} mode={mode}"), + compiler.host, + target, + ); + try_run_tests(builder, &mut cmd, false); + + if let Some(compare_mode) = compare_mode { + cmd.arg("--compare-mode").arg(compare_mode); + + #[cfg(feature = "build-metrics")] + builder.metrics.begin_test_suite( + build_helper::metrics::TestSuiteMetadata::Compiletest { + suite: suite.into(), + mode: mode.into(), + compare_mode: Some(compare_mode.into()), + target: self.target.triple.to_string(), + host: self.compiler.host.triple.to_string(), + stage: self.compiler.stage, + }, + builder, + ); + + builder.info(&format!( + "Check compiletest suite={} mode={} compare_mode={} ({} -> {})", + suite, mode, compare_mode, &compiler.host, target + )); + let _time = helpers::timeit(&builder); + try_run_tests(builder, &mut cmd, false); + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct BookTest { + compiler: Compiler, + path: PathBuf, + name: &'static str, + is_ext_doc: bool, +} + +impl Step for BookTest { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Runs the documentation tests for a book in `src/doc`. + /// + /// This uses the `rustdoc` that sits next to `compiler`. + fn run(self, builder: &Builder<'_>) { + // External docs are different from local because: + // - Some books need pre-processing by mdbook before being tested. + // - They need to save their state to toolstate. + // - They are only tested on the "checktools" builders. + // + // The local docs are tested by default, and we don't want to pay the + // cost of building mdbook, so they use `rustdoc --test` directly. + // Also, the unstable book is special because SUMMARY.md is generated, + // so it is easier to just run `rustdoc` on its files. + if self.is_ext_doc { + self.run_ext_doc(builder); + } else { + self.run_local_doc(builder); + } + } +} + +impl BookTest { + /// This runs the equivalent of `mdbook test` (via the rustbook wrapper) + /// which in turn runs `rustdoc --test` on each file in the book. + fn run_ext_doc(self, builder: &Builder<'_>) { + let compiler = self.compiler; + + builder.ensure(compile::Std::new(compiler, compiler.host)); + + // mdbook just executes a binary named "rustdoc", so we need to update + // PATH so that it points to our rustdoc. + let mut rustdoc_path = builder.rustdoc(compiler); + rustdoc_path.pop(); + let old_path = env::var_os("PATH").unwrap_or_default(); + let new_path = env::join_paths(iter::once(rustdoc_path).chain(env::split_paths(&old_path))) + .expect("could not add rustdoc to PATH"); + + let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook); + let path = builder.src.join(&self.path); + // Books often have feature-gated example text. + rustbook_cmd.env("RUSTC_BOOTSTRAP", "1"); + rustbook_cmd.env("PATH", new_path).arg("test").arg(path); + builder.add_rust_test_threads(&mut rustbook_cmd); + let _guard = builder.msg( + Kind::Test, + compiler.stage, + format_args!("mdbook {}", self.path.display()), + compiler.host, + compiler.host, + ); + let _time = helpers::timeit(&builder); + let toolstate = if builder.run_delaying_failure(&mut rustbook_cmd) { + ToolState::TestPass + } else { + ToolState::TestFail + }; + builder.save_toolstate(self.name, toolstate); + } + + /// This runs `rustdoc --test` on all `.md` files in the path. + fn run_local_doc(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let host = self.compiler.host; + + builder.ensure(compile::Std::new(compiler, host)); + + let _guard = + builder.msg(Kind::Test, compiler.stage, &format!("book {}", self.name), host, host); + + // Do a breadth-first traversal of the `src/doc` directory and just run + // tests for all files that end in `*.md` + let mut stack = vec![builder.src.join(self.path)]; + let _time = helpers::timeit(&builder); + let mut files = Vec::new(); + while let Some(p) = stack.pop() { + if p.is_dir() { + stack.extend(t!(p.read_dir()).map(|p| t!(p).path())); + continue; + } + + if p.extension().and_then(|s| s.to_str()) != Some("md") { + continue; + } + + files.push(p); + } + + files.sort(); + + for file in files { + markdown_test(builder, compiler, &file); + } + } +} + +macro_rules! test_book { + ($($name:ident, $path:expr, $book_name:expr, default=$default:expr;)+) => { + $( + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] + pub struct $name { + compiler: Compiler, + } + + impl Step for $name { + type Output = (); + const DEFAULT: bool = $default; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path($path) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure($name { + compiler: run.builder.compiler(run.builder.top_stage, run.target), + }); + } + + fn run(self, builder: &Builder<'_>) { + builder.ensure(BookTest { + compiler: self.compiler, + path: PathBuf::from($path), + name: $book_name, + is_ext_doc: !$default, + }); + } + } + )+ + } +} + +test_book!( + Nomicon, "src/doc/nomicon", "nomicon", default=false; + Reference, "src/doc/reference", "reference", default=false; + RustdocBook, "src/doc/rustdoc", "rustdoc", default=true; + RustcBook, "src/doc/rustc", "rustc", default=true; + RustByExample, "src/doc/rust-by-example", "rust-by-example", default=false; + EmbeddedBook, "src/doc/embedded-book", "embedded-book", default=false; + TheBook, "src/doc/book", "book", default=false; + UnstableBook, "src/doc/unstable-book", "unstable-book", default=true; + EditionGuide, "src/doc/edition-guide", "edition-guide", default=false; +); + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct ErrorIndex { + compiler: Compiler, +} + +impl Step for ErrorIndex { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/error_index_generator") + } + + fn make_run(run: RunConfig<'_>) { + // error_index_generator depends on librustdoc. Use the compiler that + // is normally used to build rustdoc for other tests (like compiletest + // tests in tests/rustdoc) so that it shares the same artifacts. + let compiler = run.builder.compiler(run.builder.top_stage, run.builder.config.build); + run.builder.ensure(ErrorIndex { compiler }); + } + + /// Runs the error index generator tool to execute the tests located in the error + /// index. + /// + /// The `error_index_generator` tool lives in `src/tools` and is used to + /// generate a markdown file from the error indexes of the code base which is + /// then passed to `rustdoc --test`. + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + + let dir = testdir(builder, compiler.host); + t!(fs::create_dir_all(&dir)); + let output = dir.join("error-index.md"); + + let mut tool = tool::ErrorIndex::command(builder); + tool.arg("markdown").arg(&output); + + let guard = + builder.msg(Kind::Test, compiler.stage, "error-index", compiler.host, compiler.host); + let _time = helpers::timeit(&builder); + builder.run_quiet(&mut tool); + drop(guard); + // The tests themselves need to link to std, so make sure it is + // available. + builder.ensure(compile::Std::new(compiler, compiler.host)); + markdown_test(builder, compiler, &output); + } +} + +fn markdown_test(builder: &Builder<'_>, compiler: Compiler, markdown: &Path) -> bool { + if let Ok(contents) = fs::read_to_string(markdown) { + if !contents.contains("```") { + return true; + } + } + + builder.verbose(&format!("doc tests for: {}", markdown.display())); + let mut cmd = builder.rustdoc_cmd(compiler); + builder.add_rust_test_threads(&mut cmd); + // allow for unstable options such as new editions + cmd.arg("-Z"); + cmd.arg("unstable-options"); + cmd.arg("--test"); + cmd.arg(markdown); + cmd.env("RUSTC_BOOTSTRAP", "1"); + + let test_args = builder.config.test_args().join(" "); + cmd.arg("--test-args").arg(test_args); + + if builder.config.verbose_tests { + builder.run_delaying_failure(&mut cmd) + } else { + builder.run_quiet_delaying_failure(&mut cmd) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct RustcGuide; + +impl Step for RustcGuide { + type Output = (); + const DEFAULT: bool = false; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/doc/rustc-dev-guide") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustcGuide); + } + + fn run(self, builder: &Builder<'_>) { + let relative_path = Path::new("src").join("doc").join("rustc-dev-guide"); + builder.update_submodule(&relative_path); + + let src = builder.src.join(relative_path); + let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook); + let toolstate = if builder.run_delaying_failure(rustbook_cmd.arg("linkcheck").arg(&src)) { + ToolState::TestPass + } else { + ToolState::TestFail + }; + builder.save_toolstate("rustc-dev-guide", toolstate); + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CrateLibrustc { + compiler: Compiler, + target: TargetSelection, + crates: Vec>, +} + +impl Step for CrateLibrustc { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.crate_or_deps("rustc-main") + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + let host = run.build_triple(); + let compiler = builder.compiler_for(builder.top_stage, host, host); + let crates = run + .paths + .iter() + .map(|p| builder.crate_paths[&p.assert_single_path().path].clone()) + .collect(); + + builder.ensure(CrateLibrustc { compiler, target: run.target, crates }); + } + + fn run(self, builder: &Builder<'_>) { + builder.ensure(Crate { + compiler: self.compiler, + target: self.target, + mode: Mode::Rustc, + crates: self.crates, + }); + } +} + +/// Given a `cargo test` subcommand, add the appropriate flags and run it. +/// +/// Returns whether the test succeeded. +fn run_cargo_test<'a>( + cargo: impl Into, + libtest_args: &[&str], + crates: &[Interned], + primary_crate: &str, + description: impl Into>, + compiler: Compiler, + target: TargetSelection, + builder: &Builder<'_>, +) -> bool { + let mut cargo = + prepare_cargo_test(cargo, libtest_args, crates, primary_crate, compiler, target, builder); + let _time = helpers::timeit(&builder); + let _group = description.into().and_then(|what| { + builder.msg_sysroot_tool(Kind::Test, compiler.stage, what, compiler.host, target) + }); + + #[cfg(feature = "build-metrics")] + builder.metrics.begin_test_suite( + build_helper::metrics::TestSuiteMetadata::CargoPackage { + crates: crates.iter().map(|c| c.to_string()).collect(), + target: target.triple.to_string(), + host: compiler.host.triple.to_string(), + stage: compiler.stage, + }, + builder, + ); + add_flags_and_try_run_tests(builder, &mut cargo) +} + +/// Given a `cargo test` subcommand, pass it the appropriate test flags given a `builder`. +fn prepare_cargo_test( + cargo: impl Into, + libtest_args: &[&str], + crates: &[Interned], + primary_crate: &str, + compiler: Compiler, + target: TargetSelection, + builder: &Builder<'_>, +) -> Command { + let mut cargo = cargo.into(); + + // Propegate `--bless` if it has not already been set/unset + // Any tools that want to use this should bless if `RUSTC_BLESS` is set to + // anything other than `0`. + if builder.config.cmd.bless() && !cargo.get_envs().any(|v| v.0 == "RUSTC_BLESS") { + cargo.env("RUSTC_BLESS", "Gesundheit"); + } + + // Pass in some standard flags then iterate over the graph we've discovered + // in `cargo metadata` with the maps above and figure out what `-p` + // arguments need to get passed. + if builder.kind == Kind::Test && !builder.fail_fast { + cargo.arg("--no-fail-fast"); + } + match builder.doc_tests { + DocTests::Only => { + cargo.arg("--doc"); + } + DocTests::No => { + let krate = &builder + .crates + .get(&INTERNER.intern_str(primary_crate)) + .unwrap_or_else(|| panic!("missing crate {primary_crate}")); + if krate.has_lib { + cargo.arg("--lib"); + } + cargo.args(&["--bins", "--examples", "--tests", "--benches"]); + } + DocTests::Yes => {} + } + + for &krate in crates { + cargo.arg("-p").arg(krate); + } + + cargo.arg("--").args(&builder.config.test_args()).args(libtest_args); + if !builder.config.verbose_tests { + cargo.arg("--quiet"); + } + + // The tests are going to run with the *target* libraries, so we need to + // ensure that those libraries show up in the LD_LIBRARY_PATH equivalent. + // + // Note that to run the compiler we need to run with the *host* libraries, + // but our wrapper scripts arrange for that to be the case anyway. + let mut dylib_path = dylib_path(); + dylib_path.insert(0, PathBuf::from(&*builder.sysroot_libdir(compiler, target))); + cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); + + if target.contains("emscripten") { + cargo.env( + format!("CARGO_TARGET_{}_RUNNER", envify(&target.triple)), + builder.config.nodejs.as_ref().expect("nodejs not configured"), + ); + } else if target.starts_with("wasm32") { + let node = builder.config.nodejs.as_ref().expect("nodejs not configured"); + let runner = format!("{} {}/src/etc/wasm32-shim.js", node.display(), builder.src.display()); + cargo.env(format!("CARGO_TARGET_{}_RUNNER", envify(&target.triple)), &runner); + } else if builder.remote_tested(target) { + cargo.env( + format!("CARGO_TARGET_{}_RUNNER", envify(&target.triple)), + format!("{} run 0", builder.tool_exe(Tool::RemoteTestClient).display()), + ); + } + + cargo +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Crate { + pub compiler: Compiler, + pub target: TargetSelection, + pub mode: Mode, + pub crates: Vec>, +} + +impl Step for Crate { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.crate_or_deps("sysroot") + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + let host = run.build_triple(); + let compiler = builder.compiler_for(builder.top_stage, host, host); + let crates = run + .paths + .iter() + .map(|p| builder.crate_paths[&p.assert_single_path().path].clone()) + .collect(); + + builder.ensure(Crate { compiler, target: run.target, mode: Mode::Std, crates }); + } + + /// Runs all unit tests plus documentation tests for a given crate defined + /// by a `Cargo.toml` (single manifest) + /// + /// This is what runs tests for crates like the standard library, compiler, etc. + /// It essentially is the driver for running `cargo test`. + /// + /// Currently this runs all tests for a DAG by passing a bunch of `-p foo` + /// arguments, and those arguments are discovered from `cargo metadata`. + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target = self.target; + let mode = self.mode; + + // See [field@compile::Std::force_recompile]. + builder.ensure(compile::Std::force_recompile(compiler, target)); + builder.ensure(RemoteCopyLibs { compiler, target }); + + // If we're not doing a full bootstrap but we're testing a stage2 + // version of libstd, then what we're actually testing is the libstd + // produced in stage1. Reflect that here by updating the compiler that + // we're working with automatically. + let compiler = builder.compiler_for(compiler.stage, compiler.host, target); + + let mut cargo = + builder.cargo(compiler, mode, SourceType::InTree, target, builder.kind.as_str()); + match mode { + Mode::Std => { + compile::std_cargo(builder, target, compiler.stage, &mut cargo); + // `std_cargo` actually does the wrong thing: it passes `--sysroot build/host/stage2`, + // but we want to use the force-recompile std we just built in `build/host/stage2-test-sysroot`. + // Override it. + if builder.download_rustc() && compiler.stage > 0 { + let sysroot = builder + .out + .join(compiler.host.triple) + .join(format!("stage{}-test-sysroot", compiler.stage)); + cargo.env("RUSTC_SYSROOT", sysroot); + } + } + Mode::Rustc => { + compile::rustc_cargo(builder, &mut cargo, target, compiler.stage); + } + _ => panic!("can only test libraries"), + }; + + run_cargo_test( + cargo, + &[], + &self.crates, + &self.crates[0], + &*crate_description(&self.crates), + compiler, + target, + builder, + ); + } +} + +/// Rustdoc is special in various ways, which is why this step is different from `Crate`. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct CrateRustdoc { + host: TargetSelection, +} + +impl Step for CrateRustdoc { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.paths(&["src/librustdoc", "src/tools/rustdoc"]) + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + + builder.ensure(CrateRustdoc { host: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let target = self.host; + + let compiler = if builder.download_rustc() { + builder.compiler(builder.top_stage, target) + } else { + // Use the previous stage compiler to reuse the artifacts that are + // created when running compiletest for tests/rustdoc. If this used + // `compiler`, then it would cause rustdoc to be built *again*, which + // isn't really necessary. + builder.compiler_for(builder.top_stage, target, target) + }; + // NOTE: normally `ensure(Rustc)` automatically runs `ensure(Std)` for us. However, when + // using `download-rustc`, the rustc_private artifacts may be in a *different sysroot* from + // the target rustdoc (`ci-rustc-sysroot` vs `stage2`). In that case, we need to ensure this + // explicitly to make sure it ends up in the stage2 sysroot. + builder.ensure(compile::Std::new(compiler, target)); + builder.ensure(compile::Rustc::new(compiler, target)); + + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + target, + builder.kind.as_str(), + "src/tools/rustdoc", + SourceType::InTree, + &[], + ); + if self.host.contains("musl") { + cargo.arg("'-Ctarget-feature=-crt-static'"); + } + + // This is needed for running doctests on librustdoc. This is a bit of + // an unfortunate interaction with how bootstrap works and how cargo + // sets up the dylib path, and the fact that the doctest (in + // html/markdown.rs) links to rustc-private libs. For stage1, the + // compiler host dylibs (in stage1/lib) are not the same as the target + // dylibs (in stage1/lib/rustlib/...). This is different from a normal + // rust distribution where they are the same. + // + // On the cargo side, normal tests use `target_process` which handles + // setting up the dylib for a *target* (stage1/lib/rustlib/... in this + // case). However, for doctests it uses `rustdoc_process` which only + // sets up the dylib path for the *host* (stage1/lib), which is the + // wrong directory. + // + // Recall that we special-cased `compiler_for(top_stage)` above, so we always use stage1. + // + // It should be considered to just stop running doctests on + // librustdoc. There is only one test, and it doesn't look too + // important. There might be other ways to avoid this, but it seems + // pretty convoluted. + // + // See also https://github.com/rust-lang/rust/issues/13983 where the + // host vs target dylibs for rustdoc are consistently tricky to deal + // with. + // + // Note that this set the host libdir for `download_rustc`, which uses a normal rust distribution. + let libdir = if builder.download_rustc() { + builder.rustc_libdir(compiler) + } else { + builder.sysroot_libdir(compiler, target).to_path_buf() + }; + let mut dylib_path = dylib_path(); + dylib_path.insert(0, PathBuf::from(&*libdir)); + cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); + + run_cargo_test( + cargo, + &[], + &[INTERNER.intern_str("rustdoc:0.0.0")], + "rustdoc", + "rustdoc", + compiler, + target, + builder, + ); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct CrateRustdocJsonTypes { + host: TargetSelection, +} + +impl Step for CrateRustdocJsonTypes { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/rustdoc-json-types") + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + + builder.ensure(CrateRustdocJsonTypes { host: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let target = self.host; + + // Use the previous stage compiler to reuse the artifacts that are + // created when running compiletest for tests/rustdoc. If this used + // `compiler`, then it would cause rustdoc to be built *again*, which + // isn't really necessary. + let compiler = builder.compiler_for(builder.top_stage, target, target); + builder.ensure(compile::Rustc::new(compiler, target)); + + let cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + target, + builder.kind.as_str(), + "src/rustdoc-json-types", + SourceType::InTree, + &[], + ); + + // FIXME: this looks very wrong, libtest doesn't accept `-C` arguments and the quotes are fishy. + let libtest_args = if self.host.contains("musl") { + ["'-Ctarget-feature=-crt-static'"].as_slice() + } else { + &[] + }; + + run_cargo_test( + cargo, + libtest_args, + &[INTERNER.intern_str("rustdoc-json-types")], + "rustdoc-json-types", + "rustdoc-json-types", + compiler, + target, + builder, + ); + } +} + +/// Some test suites are run inside emulators or on remote devices, and most +/// of our test binaries are linked dynamically which means we need to ship +/// the standard library and such to the emulator ahead of time. This step +/// represents this and is a dependency of all test suites. +/// +/// Most of the time this is a no-op. For some steps such as shipping data to +/// QEMU we have to build our own tools so we've got conditional dependencies +/// on those programs as well. Note that the remote test client is built for +/// the build target (us) and the server is built for the target. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct RemoteCopyLibs { + compiler: Compiler, + target: TargetSelection, +} + +impl Step for RemoteCopyLibs { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target = self.target; + if !builder.remote_tested(target) { + return; + } + + builder.ensure(compile::Std::new(compiler, target)); + + builder.info(&format!("REMOTE copy libs to emulator ({target})")); + + let server = builder.ensure(tool::RemoteTestServer { compiler, target }); + + // Spawn the emulator and wait for it to come online + let tool = builder.tool_exe(Tool::RemoteTestClient); + let mut cmd = Command::new(&tool); + cmd.arg("spawn-emulator").arg(target.triple).arg(&server).arg(builder.tempdir()); + if let Some(rootfs) = builder.qemu_rootfs(target) { + cmd.arg(rootfs); + } + builder.run(&mut cmd); + + // Push all our dylibs to the emulator + for f in t!(builder.sysroot_libdir(compiler, target).read_dir()) { + let f = t!(f); + let name = f.file_name().into_string().unwrap(); + if helpers::is_dylib(&name) { + builder.run(Command::new(&tool).arg("push").arg(f.path())); + } + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Distcheck; + +impl Step for Distcheck { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("distcheck") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Distcheck); + } + + /// Runs "distcheck", a 'make check' from a tarball + fn run(self, builder: &Builder<'_>) { + builder.info("Distcheck"); + let dir = builder.tempdir().join("distcheck"); + let _ = fs::remove_dir_all(&dir); + t!(fs::create_dir_all(&dir)); + + // Guarantee that these are built before we begin running. + builder.ensure(dist::PlainSourceTarball); + builder.ensure(dist::Src); + + let mut cmd = Command::new("tar"); + cmd.arg("-xf") + .arg(builder.ensure(dist::PlainSourceTarball).tarball()) + .arg("--strip-components=1") + .current_dir(&dir); + builder.run(&mut cmd); + builder.run( + Command::new("./configure") + .args(&builder.config.configure_args) + .arg("--enable-vendor") + .current_dir(&dir), + ); + builder.run( + Command::new(helpers::make(&builder.config.build.triple)) + .arg("check") + .current_dir(&dir), + ); + + // Now make sure that rust-src has all of libstd's dependencies + builder.info("Distcheck rust-src"); + let dir = builder.tempdir().join("distcheck-src"); + let _ = fs::remove_dir_all(&dir); + t!(fs::create_dir_all(&dir)); + + let mut cmd = Command::new("tar"); + cmd.arg("-xf") + .arg(builder.ensure(dist::Src).tarball()) + .arg("--strip-components=1") + .current_dir(&dir); + builder.run(&mut cmd); + + let toml = dir.join("rust-src/lib/rustlib/src/rust/library/std/Cargo.toml"); + builder.run( + Command::new(&builder.initial_cargo) + // Will read the libstd Cargo.toml + // which uses the unstable `public-dependency` feature. + .env("RUSTC_BOOTSTRAP", "1") + .arg("generate-lockfile") + .arg("--manifest-path") + .arg(&toml) + .current_dir(&dir), + ); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Bootstrap; + +impl Step for Bootstrap { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + /// Tests the build system itself. + fn run(self, builder: &Builder<'_>) { + let host = builder.config.build; + let compiler = builder.compiler(0, host); + let _guard = builder.msg(Kind::Test, 0, "bootstrap", host, host); + + let mut check_bootstrap = Command::new(&builder.python()); + check_bootstrap + .args(["-m", "unittest", "bootstrap_test.py"]) + .env("BUILD_DIR", &builder.out) + .env("BUILD_PLATFORM", &builder.build.build.triple) + .current_dir(builder.src.join("src/bootstrap/")); + // NOTE: we intentionally don't pass test_args here because the args for unittest and cargo test are mutually incompatible. + // Use `python -m unittest` manually if you want to pass arguments. + builder.run_delaying_failure(&mut check_bootstrap); + + let mut cmd = Command::new(&builder.initial_cargo); + cmd.arg("test") + .current_dir(builder.src.join("src/bootstrap")) + .env("RUSTFLAGS", "-Cdebuginfo=2") + .env("CARGO_TARGET_DIR", builder.out.join("bootstrap")) + .env("RUSTC_BOOTSTRAP", "1") + .env("RUSTDOC", builder.rustdoc(compiler)) + .env("RUSTC", &builder.initial_rustc); + if let Some(flags) = option_env!("RUSTFLAGS") { + // Use the same rustc flags for testing as for "normal" compilation, + // so that Cargo doesn’t recompile the entire dependency graph every time: + // https://github.com/rust-lang/rust/issues/49215 + cmd.env("RUSTFLAGS", flags); + } + // rustbuild tests are racy on directory creation so just run them one at a time. + // Since there's not many this shouldn't be a problem. + run_cargo_test(cmd, &["--test-threads=1"], &[], "bootstrap", None, compiler, host, builder); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/bootstrap") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Bootstrap); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct TierCheck { + pub compiler: Compiler, +} + +impl Step for TierCheck { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/tier-check") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = + run.builder.compiler_for(run.builder.top_stage, run.builder.build.build, run.target); + run.builder.ensure(TierCheck { compiler }); + } + + /// Tests the Platform Support page in the rustc book. + fn run(self, builder: &Builder<'_>) { + builder.ensure(compile::Std::new(self.compiler, self.compiler.host)); + let mut cargo = tool::prepare_tool_cargo( + builder, + self.compiler, + Mode::ToolStd, + self.compiler.host, + "run", + "src/tools/tier-check", + SourceType::InTree, + &[], + ); + cargo.arg(builder.src.join("src/doc/rustc/src/platform-support.md")); + cargo.arg(&builder.rustc(self.compiler)); + if builder.is_verbose() { + cargo.arg("--verbose"); + } + + let _guard = builder.msg( + Kind::Test, + self.compiler.stage, + "platform support check", + self.compiler.host, + self.compiler.host, + ); + builder.run_delaying_failure(&mut cargo.into()); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct LintDocs { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for LintDocs { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/lint-docs") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(LintDocs { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + /// Tests that the lint examples in the rustc book generate the correct + /// lints and have the expected format. + fn run(self, builder: &Builder<'_>) { + builder.ensure(crate::core::build_steps::doc::RustcBook { + compiler: self.compiler, + target: self.target, + validate: true, + }); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct RustInstaller; + +impl Step for RustInstaller { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + /// Ensure the version placeholder replacement tool builds + fn run(self, builder: &Builder<'_>) { + let bootstrap_host = builder.config.build; + let compiler = builder.compiler(0, bootstrap_host); + let cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolBootstrap, + bootstrap_host, + "test", + "src/tools/rust-installer", + SourceType::InTree, + &[], + ); + + let _guard = builder.msg( + Kind::Test, + compiler.stage, + "rust-installer", + bootstrap_host, + bootstrap_host, + ); + run_cargo_test(cargo, &[], &[], "installer", None, compiler, bootstrap_host, builder); + + // We currently don't support running the test.sh script outside linux(?) environments. + // Eventually this should likely migrate to #[test]s in rust-installer proper rather than a + // set of scripts, which will likely allow dropping this if. + if bootstrap_host != "x86_64-unknown-linux-gnu" { + return; + } + + let mut cmd = + std::process::Command::new(builder.src.join("src/tools/rust-installer/test.sh")); + let tmpdir = testdir(builder, compiler.host).join("rust-installer"); + let _ = std::fs::remove_dir_all(&tmpdir); + let _ = std::fs::create_dir_all(&tmpdir); + cmd.current_dir(&tmpdir); + cmd.env("CARGO_TARGET_DIR", tmpdir.join("cargo-target")); + cmd.env("CARGO", &builder.initial_cargo); + cmd.env("RUSTC", &builder.initial_rustc); + cmd.env("TMP_DIR", &tmpdir); + builder.run_delaying_failure(&mut cmd); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rust-installer") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Self); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct TestHelpers { + pub target: TargetSelection, +} + +impl Step for TestHelpers { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("tests/auxiliary/rust_test_helpers.c") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(TestHelpers { target: run.target }) + } + + /// Compiles the `rust_test_helpers.c` library which we used in various + /// `run-pass` tests for ABI testing. + fn run(self, builder: &Builder<'_>) { + if builder.config.dry_run() { + return; + } + // The x86_64-fortanix-unknown-sgx target doesn't have a working C + // toolchain. However, some x86_64 ELF objects can be linked + // without issues. Use this hack to compile the test helpers. + let target = if self.target == "x86_64-fortanix-unknown-sgx" { + TargetSelection::from_user("x86_64-unknown-linux-gnu") + } else { + self.target + }; + let dst = builder.test_helpers_out(target); + let src = builder.src.join("tests/auxiliary/rust_test_helpers.c"); + if up_to_date(&src, &dst.join("librust_test_helpers.a")) { + return; + } + + let _guard = builder.msg_unstaged(Kind::Build, "test helpers", target); + t!(fs::create_dir_all(&dst)); + let mut cfg = cc::Build::new(); + + // We may have found various cross-compilers a little differently due to our + // extra configuration, so inform cc of these compilers. Note, though, that + // on MSVC we still need cc's detection of env vars (ugh). + if !target.contains("msvc") { + if let Some(ar) = builder.ar(target) { + cfg.archiver(ar); + } + cfg.compiler(builder.cc(target)); + } + cfg.cargo_metadata(false) + .out_dir(&dst) + .target(&target.triple) + .host(&builder.config.build.triple) + .opt_level(0) + .warnings(false) + .debug(false) + .file(builder.src.join("tests/auxiliary/rust_test_helpers.c")) + .compile("rust_test_helpers"); + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CodegenCranelift { + compiler: Compiler, + target: TargetSelection, +} + +impl Step for CodegenCranelift { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.paths(&["compiler/rustc_codegen_cranelift"]) + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + let host = run.build_triple(); + let compiler = run.builder.compiler_for(run.builder.top_stage, host, host); + + if builder.doc_tests == DocTests::Only { + return; + } + + if !target_supports_cranelift_backend(run.target) { + builder.info("target not supported by rustc_codegen_cranelift. skipping"); + return; + } + + if builder.remote_tested(run.target) { + builder.info("remote testing is not supported by rustc_codegen_cranelift. skipping"); + return; + } + + if !builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("cranelift")) { + builder.info("cranelift not in rust.codegen-backends. skipping"); + return; + } + + builder.ensure(CodegenCranelift { compiler, target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target = self.target; + + builder.ensure(compile::Std::new(compiler, target)); + + // If we're not doing a full bootstrap but we're testing a stage2 + // version of libstd, then what we're actually testing is the libstd + // produced in stage1. Reflect that here by updating the compiler that + // we're working with automatically. + let compiler = builder.compiler_for(compiler.stage, compiler.host, target); + + let build_cargo = || { + let mut cargo = builder.cargo( + compiler, + Mode::Codegen, // Must be codegen to ensure dlopen on compiled dylibs works + SourceType::InTree, + target, + "run", + ); + cargo.current_dir(&builder.src.join("compiler/rustc_codegen_cranelift")); + cargo + .arg("--manifest-path") + .arg(builder.src.join("compiler/rustc_codegen_cranelift/build_system/Cargo.toml")); + compile::rustc_cargo_env(builder, &mut cargo, target, compiler.stage); + + // Avoid incremental cache issues when changing rustc + cargo.env("CARGO_BUILD_INCREMENTAL", "false"); + + cargo + }; + + builder.info(&format!( + "{} cranelift stage{} ({} -> {})", + Kind::Test.description(), + compiler.stage, + &compiler.host, + target + )); + let _time = helpers::timeit(&builder); + + // FIXME handle vendoring for source tarballs before removing the --skip-test below + let download_dir = builder.out.join("cg_clif_download"); + + // FIXME: Uncomment the `prepare` command below once vendoring is implemented. + /* + let mut prepare_cargo = build_cargo(); + prepare_cargo.arg("--").arg("prepare").arg("--download-dir").arg(&download_dir); + #[allow(deprecated)] + builder.config.try_run(&mut prepare_cargo.into()).unwrap(); + */ + + let mut cargo = build_cargo(); + cargo + .arg("--") + .arg("test") + .arg("--download-dir") + .arg(&download_dir) + .arg("--out-dir") + .arg(builder.stage_out(compiler, Mode::ToolRustc).join("cg_clif")) + .arg("--no-unstable-features") + .arg("--use-backend") + .arg("cranelift") + // Avoid having to vendor the standard library dependencies + .arg("--sysroot") + .arg("llvm") + // These tests depend on crates that are not yet vendored + // FIXME remove once vendoring is handled + .arg("--skip-test") + .arg("testsuite.extended_sysroot"); + cargo.args(builder.config.test_args()); + + let mut cmd: Command = cargo.into(); + builder.run_cmd(BootstrapCommand::from(&mut cmd).fail_fast()); + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CodegenGCC { + compiler: Compiler, + target: TargetSelection, +} + +impl Step for CodegenGCC { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.paths(&["compiler/rustc_codegen_gcc"]) + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + let host = run.build_triple(); + let compiler = run.builder.compiler_for(run.builder.top_stage, host, host); + + if builder.doc_tests == DocTests::Only { + return; + } + + let triple = run.target.triple; + let target_supported = + if triple.contains("linux") { triple.contains("x86_64") } else { false }; + if !target_supported { + builder.info("target not supported by rustc_codegen_gcc. skipping"); + return; + } + + if builder.remote_tested(run.target) { + builder.info("remote testing is not supported by rustc_codegen_gcc. skipping"); + return; + } + + if !builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("gcc")) { + builder.info("gcc not in rust.codegen-backends. skipping"); + return; + } + + builder.ensure(CodegenGCC { compiler, target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target = self.target; + + builder.ensure(compile::Std::new_with_extra_rust_args( + compiler, + target, + &["-Csymbol-mangling-version=v0", "-Cpanic=abort"], + )); + + // If we're not doing a full bootstrap but we're testing a stage2 + // version of libstd, then what we're actually testing is the libstd + // produced in stage1. Reflect that here by updating the compiler that + // we're working with automatically. + let compiler = builder.compiler_for(compiler.stage, compiler.host, target); + + let build_cargo = || { + let mut cargo = builder.cargo( + compiler, + Mode::Codegen, // Must be codegen to ensure dlopen on compiled dylibs works + SourceType::InTree, + target, + "run", + ); + cargo.current_dir(&builder.src.join("compiler/rustc_codegen_gcc")); + cargo + .arg("--manifest-path") + .arg(builder.src.join("compiler/rustc_codegen_gcc/build_system/Cargo.toml")); + compile::rustc_cargo_env(builder, &mut cargo, target, compiler.stage); + + // Avoid incremental cache issues when changing rustc + cargo.env("CARGO_BUILD_INCREMENTAL", "false"); + cargo.rustflag("-Cpanic=abort"); + + cargo + }; + + builder.info(&format!( + "{} GCC stage{} ({} -> {})", + Kind::Test.description(), + compiler.stage, + &compiler.host, + target + )); + let _time = helpers::timeit(&builder); + + // FIXME: Uncomment the `prepare` command below once vendoring is implemented. + /* + let mut prepare_cargo = build_cargo(); + prepare_cargo.arg("--").arg("prepare"); + #[allow(deprecated)] + builder.config.try_run(&mut prepare_cargo.into()).unwrap(); + */ + + let mut cargo = build_cargo(); + + cargo + .arg("--") + .arg("test") + .arg("--use-system-gcc") + .arg("--use-backend") + .arg("gcc") + .arg("--out-dir") + .arg(builder.stage_out(compiler, Mode::ToolRustc).join("cg_gcc")) + .arg("--release") + .arg("--no-default-features") + .arg("--mini-tests") + .arg("--std-tests"); + cargo.args(builder.config.test_args()); + + let mut cmd: Command = cargo.into(); + builder.run_cmd(BootstrapCommand::from(&mut cmd).fail_fast()); + } +} diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs new file mode 100644 index 000000000..d1bc05e51 --- /dev/null +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -0,0 +1,849 @@ +use std::env; +use std::fs; +use std::path::PathBuf; +use std::process::Command; + +use crate::core::build_steps::compile; +use crate::core::build_steps::toolstate::ToolState; +use crate::core::builder::{Builder, Cargo as CargoCommand, RunConfig, ShouldRun, Step}; +use crate::core::config::TargetSelection; +use crate::utils::channel::GitInfo; +use crate::utils::exec::BootstrapCommand; +use crate::utils::helpers::{add_dylib_path, exe, t}; +use crate::Compiler; +use crate::Mode; +use crate::{gha, Kind}; + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub enum SourceType { + InTree, + Submodule, +} + +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +struct ToolBuild { + compiler: Compiler, + target: TargetSelection, + tool: &'static str, + path: &'static str, + mode: Mode, + is_optional_tool: bool, + source_type: SourceType, + extra_features: Vec, + /// Nightly-only features that are allowed (comma-separated list). + allow_features: &'static str, +} + +impl Builder<'_> { + #[track_caller] + fn msg_tool( + &self, + mode: Mode, + tool: &str, + build_stage: u32, + host: &TargetSelection, + target: &TargetSelection, + ) -> Option { + match mode { + // depends on compiler stage, different to host compiler + Mode::ToolRustc => self.msg_sysroot_tool( + Kind::Build, + build_stage, + format_args!("tool {tool}"), + *host, + *target, + ), + // doesn't depend on compiler, same as host compiler + _ => self.msg(Kind::Build, build_stage, format_args!("tool {tool}"), *host, *target), + } + } +} + +impl Step for ToolBuild { + type Output = Option; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Builds a tool in `src/tools` + /// + /// This will build the specified tool with the specified `host` compiler in + /// `stage` into the normal cargo output directory. + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + let mut tool = self.tool; + let path = self.path; + let is_optional_tool = self.is_optional_tool; + + match self.mode { + Mode::ToolRustc => { + builder.ensure(compile::Std::new(compiler, compiler.host)); + builder.ensure(compile::Rustc::new(compiler, target)); + } + Mode::ToolStd => builder.ensure(compile::Std::new(compiler, target)), + Mode::ToolBootstrap => {} // uses downloaded stage0 compiler libs + _ => panic!("unexpected Mode for tool build"), + } + + let mut cargo = prepare_tool_cargo( + builder, + compiler, + self.mode, + target, + "build", + path, + self.source_type, + &self.extra_features, + ); + if !self.allow_features.is_empty() { + cargo.allow_features(self.allow_features); + } + let _guard = builder.msg_tool( + self.mode, + self.tool, + self.compiler.stage, + &self.compiler.host, + &self.target, + ); + + let mut cargo = Command::from(cargo); + // we check this in `is_optional_tool` in a second + let is_expected = builder.run_cmd(BootstrapCommand::from(&mut cargo).allow_failure()); + + builder.save_toolstate( + tool, + if is_expected { ToolState::TestFail } else { ToolState::BuildFail }, + ); + + if !is_expected { + if !is_optional_tool { + crate::exit!(1); + } else { + None + } + } else { + // HACK(#82501): on Windows, the tools directory gets added to PATH when running tests, and + // compiletest confuses HTML tidy with the in-tree tidy. Name the in-tree tidy something + // different so the problem doesn't come up. + if tool == "tidy" { + tool = "rust-tidy"; + } + let cargo_out = builder.cargo_out(compiler, self.mode, target).join(exe(tool, target)); + let bin = builder.tools_dir(compiler).join(exe(tool, target)); + builder.copy(&cargo_out, &bin); + Some(bin) + } + } +} + +pub fn prepare_tool_cargo( + builder: &Builder<'_>, + compiler: Compiler, + mode: Mode, + target: TargetSelection, + command: &'static str, + path: &str, + source_type: SourceType, + extra_features: &[String], +) -> CargoCommand { + let mut cargo = builder.cargo(compiler, mode, source_type, target, command); + let dir = builder.src.join(path); + cargo.arg("--manifest-path").arg(dir.join("Cargo.toml")); + + let mut features = extra_features.to_vec(); + if builder.build.config.cargo_native_static { + if path.ends_with("cargo") + || path.ends_with("rls") + || path.ends_with("clippy") + || path.ends_with("miri") + || path.ends_with("rustfmt") + { + cargo.env("LIBZ_SYS_STATIC", "1"); + } + if path.ends_with("cargo") { + features.push("all-static".to_string()); + } + } + + // clippy tests need to know about the stage sysroot. Set them consistently while building to + // avoid rebuilding when running tests. + cargo.env("SYSROOT", builder.sysroot(compiler)); + + // if tools are using lzma we want to force the build script to build its + // own copy + cargo.env("LZMA_API_STATIC", "1"); + + // CFG_RELEASE is needed by rustfmt (and possibly other tools) which + // import rustc-ap-rustc_attr which requires this to be set for the + // `#[cfg(version(...))]` attribute. + cargo.env("CFG_RELEASE", builder.rust_release()); + cargo.env("CFG_RELEASE_CHANNEL", &builder.config.channel); + cargo.env("CFG_VERSION", builder.rust_version()); + cargo.env("CFG_RELEASE_NUM", &builder.version); + cargo.env("DOC_RUST_LANG_ORG_CHANNEL", builder.doc_rust_lang_org_channel()); + if let Some(ref ver_date) = builder.rust_info().commit_date() { + cargo.env("CFG_VER_DATE", ver_date); + } + if let Some(ref ver_hash) = builder.rust_info().sha() { + cargo.env("CFG_VER_HASH", ver_hash); + } + + let info = GitInfo::new(builder.config.omit_git_hash, &dir); + if let Some(sha) = info.sha() { + cargo.env("CFG_COMMIT_HASH", sha); + } + if let Some(sha_short) = info.sha_short() { + cargo.env("CFG_SHORT_COMMIT_HASH", sha_short); + } + if let Some(date) = info.commit_date() { + cargo.env("CFG_COMMIT_DATE", date); + } + if !features.is_empty() { + cargo.arg("--features").arg(&features.join(", ")); + } + cargo +} + +macro_rules! bootstrap_tool { + ($( + $name:ident, $path:expr, $tool_name:expr + $(,is_external_tool = $external:expr)* + $(,is_unstable_tool = $unstable:expr)* + $(,allow_features = $allow_features:expr)? + ; + )+) => { + #[derive(Copy, PartialEq, Eq, Clone)] + pub enum Tool { + $( + $name, + )+ + } + + impl<'a> Builder<'a> { + pub fn tool_exe(&self, tool: Tool) -> PathBuf { + match tool { + $(Tool::$name => + self.ensure($name { + compiler: self.compiler(0, self.config.build), + target: self.config.build, + }), + )+ + } + } + } + + $( + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] + pub struct $name { + pub compiler: Compiler, + pub target: TargetSelection, + } + + impl Step for $name { + type Output = PathBuf; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path($path) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure($name { + // snapshot compiler + compiler: run.builder.compiler(0, run.builder.config.build), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: $tool_name, + mode: if false $(|| $unstable)* { + // use in-tree libraries for unstable features + Mode::ToolStd + } else { + Mode::ToolBootstrap + }, + path: $path, + is_optional_tool: false, + source_type: if false $(|| $external)* { + SourceType::Submodule + } else { + SourceType::InTree + }, + extra_features: vec![], + allow_features: concat!($($allow_features)*), + }).expect("expected to build -- essential tool") + } + } + )+ + } +} + +bootstrap_tool!( + Rustbook, "src/tools/rustbook", "rustbook"; + UnstableBookGen, "src/tools/unstable-book-gen", "unstable-book-gen"; + Tidy, "src/tools/tidy", "tidy"; + Linkchecker, "src/tools/linkchecker", "linkchecker"; + CargoTest, "src/tools/cargotest", "cargotest"; + Compiletest, "src/tools/compiletest", "compiletest", is_unstable_tool = true, allow_features = "test"; + BuildManifest, "src/tools/build-manifest", "build-manifest"; + RemoteTestClient, "src/tools/remote-test-client", "remote-test-client"; + RustInstaller, "src/tools/rust-installer", "rust-installer"; + RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes"; + ExpandYamlAnchors, "src/tools/expand-yaml-anchors", "expand-yaml-anchors"; + LintDocs, "src/tools/lint-docs", "lint-docs"; + JsonDocCk, "src/tools/jsondocck", "jsondocck"; + JsonDocLint, "src/tools/jsondoclint", "jsondoclint"; + HtmlChecker, "src/tools/html-checker", "html-checker"; + BumpStage0, "src/tools/bump-stage0", "bump-stage0"; + ReplaceVersionPlaceholder, "src/tools/replace-version-placeholder", "replace-version-placeholder"; + CollectLicenseMetadata, "src/tools/collect-license-metadata", "collect-license-metadata"; + GenerateCopyright, "src/tools/generate-copyright", "generate-copyright"; + SuggestTests, "src/tools/suggest-tests", "suggest-tests"; + GenerateWindowsSys, "src/tools/generate-windows-sys", "generate-windows-sys"; + RustdocGUITest, "src/tools/rustdoc-gui-test", "rustdoc-gui-test", is_unstable_tool = true, allow_features = "test"; + OptimizedDist, "src/tools/opt-dist", "opt-dist"; + CoverageDump, "src/tools/coverage-dump", "coverage-dump"; +); + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] +pub struct ErrorIndex { + pub compiler: Compiler, +} + +impl ErrorIndex { + pub fn command(builder: &Builder<'_>) -> Command { + // Error-index-generator links with the rustdoc library, so we need to add `rustc_lib_paths` + // for rustc_private and libLLVM.so, and `sysroot_lib` for libstd, etc. + let host = builder.config.build; + let compiler = builder.compiler_for(builder.top_stage, host, host); + let mut cmd = Command::new(builder.ensure(ErrorIndex { compiler })); + let mut dylib_paths = builder.rustc_lib_paths(compiler); + dylib_paths.push(PathBuf::from(&builder.sysroot_libdir(compiler, compiler.host))); + add_dylib_path(dylib_paths, &mut cmd); + cmd + } +} + +impl Step for ErrorIndex { + type Output = PathBuf; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/error_index_generator") + } + + fn make_run(run: RunConfig<'_>) { + // Compile the error-index in the same stage as rustdoc to avoid + // recompiling rustdoc twice if we can. + // + // NOTE: This `make_run` isn't used in normal situations, only if you + // manually build the tool with `x.py build + // src/tools/error-index-generator` which almost nobody does. + // Normally, `x.py test` or `x.py doc` will use the + // `ErrorIndex::command` function instead. + let compiler = + run.builder.compiler(run.builder.top_stage.saturating_sub(1), run.builder.config.build); + run.builder.ensure(ErrorIndex { compiler }); + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + builder + .ensure(ToolBuild { + compiler: self.compiler, + target: self.compiler.host, + tool: "error_index_generator", + mode: Mode::ToolRustc, + path: "src/tools/error_index_generator", + is_optional_tool: false, + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + }) + .expect("expected to build -- essential tool") + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RemoteTestServer { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for RemoteTestServer { + type Output = PathBuf; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/remote-test-server") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RemoteTestServer { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + builder + .ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "remote-test-server", + mode: Mode::ToolStd, + path: "src/tools/remote-test-server", + is_optional_tool: false, + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + }) + .expect("expected to build -- essential tool") + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] +pub struct Rustdoc { + /// This should only ever be 0 or 2. + /// We sometimes want to reference the "bootstrap" rustdoc, which is why this option is here. + pub compiler: Compiler, +} + +impl Step for Rustdoc { + type Output = PathBuf; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rustdoc").path("src/librustdoc") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Rustdoc { + // NOTE: this is somewhat unique in that we actually want a *target* + // compiler here, because rustdoc *is* a compiler. We won't be using + // this as the compiler to build with, but rather this is "what + // compiler are we producing"? + compiler: run.builder.compiler(run.builder.top_stage, run.target), + }); + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + let target_compiler = self.compiler; + if target_compiler.stage == 0 { + if !target_compiler.is_snapshot(builder) { + panic!("rustdoc in stage 0 must be snapshot rustdoc"); + } + return builder.initial_rustc.with_file_name(exe("rustdoc", target_compiler.host)); + } + let target = target_compiler.host; + // Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise + // we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage + // compilers, which isn't what we want. Rustdoc should be linked in the same way as the + // rustc compiler it's paired with, so it must be built with the previous stage compiler. + let build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build); + + // When using `download-rustc` and a stage0 build_compiler, copying rustc doesn't actually + // build stage0 libstd (because the libstd in sysroot has the wrong ABI). Explicitly build + // it. + builder.ensure(compile::Std::new(build_compiler, target_compiler.host)); + builder.ensure(compile::Rustc::new(build_compiler, target_compiler.host)); + // NOTE: this implies that `download-rustc` is pretty useless when compiling with the stage0 + // compiler, since you do just as much work. + if !builder.config.dry_run() && builder.download_rustc() && build_compiler.stage == 0 { + println!( + "WARNING: `download-rustc` does nothing when building stage1 tools; consider using `--stage 2` instead" + ); + } + + // The presence of `target_compiler` ensures that the necessary libraries (codegen backends, + // compiler libraries, ...) are built. Rustdoc does not require the presence of any + // libraries within sysroot_libdir (i.e., rustlib), though doctests may want it (since + // they'll be linked to those libraries). As such, don't explicitly `ensure` any additional + // libraries here. The intuition here is that If we've built a compiler, we should be able + // to build rustdoc. + // + let mut features = Vec::new(); + if builder.config.jemalloc { + features.push("jemalloc".to_string()); + } + + let mut cargo = prepare_tool_cargo( + builder, + build_compiler, + Mode::ToolRustc, + target, + "build", + "src/tools/rustdoc", + SourceType::InTree, + features.as_slice(), + ); + + if builder.config.rustc_parallel { + cargo.rustflag("--cfg=parallel_compiler"); + } + + let _guard = builder.msg_tool( + Mode::ToolRustc, + "rustdoc", + build_compiler.stage, + &self.compiler.host, + &target, + ); + builder.run(&mut cargo.into()); + + // Cargo adds a number of paths to the dylib search path on windows, which results in + // the wrong rustdoc being executed. To avoid the conflicting rustdocs, we name the "tool" + // rustdoc a different name. + let tool_rustdoc = builder + .cargo_out(build_compiler, Mode::ToolRustc, target) + .join(exe("rustdoc_tool_binary", target_compiler.host)); + + // don't create a stage0-sysroot/bin directory. + if target_compiler.stage > 0 { + let sysroot = builder.sysroot(target_compiler); + let bindir = sysroot.join("bin"); + t!(fs::create_dir_all(&bindir)); + let bin_rustdoc = bindir.join(exe("rustdoc", target_compiler.host)); + let _ = fs::remove_file(&bin_rustdoc); + builder.copy(&tool_rustdoc, &bin_rustdoc); + bin_rustdoc + } else { + tool_rustdoc + } + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Cargo { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Cargo { + type Output = PathBuf; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/tools/cargo").default_condition( + builder.config.extended + && builder.config.tools.as_ref().map_or( + true, + // If `tools` is set, search list for this tool. + |tools| tools.iter().any(|tool| tool == "cargo"), + ), + ) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Cargo { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + let cargo_bin_path = builder + .ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "cargo", + mode: Mode::ToolRustc, + path: "src/tools/cargo", + is_optional_tool: false, + source_type: SourceType::Submodule, + extra_features: Vec::new(), + allow_features: "", + }) + .expect("expected to build -- essential tool"); + cargo_bin_path + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct LldWrapper { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for LldWrapper { + type Output = PathBuf; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + let src_exe = builder + .ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "lld-wrapper", + mode: Mode::ToolStd, + path: "src/tools/lld-wrapper", + is_optional_tool: false, + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + }) + .expect("expected to build -- essential tool"); + + src_exe + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustAnalyzer { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl RustAnalyzer { + pub const ALLOW_FEATURES: &'static str = + "proc_macro_internals,proc_macro_diagnostic,proc_macro_span,proc_macro_span_shrink"; +} + +impl Step for RustAnalyzer { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/tools/rust-analyzer").default_condition( + builder.config.extended + && builder + .config + .tools + .as_ref() + .map_or(true, |tools| tools.iter().any(|tool| tool == "rust-analyzer")), + ) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustAnalyzer { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "rust-analyzer", + mode: Mode::ToolStd, + path: "src/tools/rust-analyzer", + extra_features: vec!["rust-analyzer/in-rust-tree".to_owned()], + is_optional_tool: false, + source_type: SourceType::InTree, + allow_features: RustAnalyzer::ALLOW_FEATURES, + }) + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustAnalyzerProcMacroSrv { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for RustAnalyzerProcMacroSrv { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + // Allow building `rust-analyzer-proc-macro-srv` both as part of the `rust-analyzer` and as a stand-alone tool. + run.path("src/tools/rust-analyzer") + .path("src/tools/rust-analyzer/crates/proc-macro-srv-cli") + .default_condition(builder.config.tools.as_ref().map_or(true, |tools| { + tools + .iter() + .any(|tool| tool == "rust-analyzer" || tool == "rust-analyzer-proc-macro-srv") + })) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustAnalyzerProcMacroSrv { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let path = builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "rust-analyzer-proc-macro-srv", + mode: Mode::ToolStd, + path: "src/tools/rust-analyzer/crates/proc-macro-srv-cli", + extra_features: vec!["sysroot-abi".to_owned()], + is_optional_tool: false, + source_type: SourceType::InTree, + allow_features: RustAnalyzer::ALLOW_FEATURES, + })?; + + // Copy `rust-analyzer-proc-macro-srv` to `/libexec/` + // so that r-a can use it. + let libexec_path = builder.sysroot(self.compiler).join("libexec"); + t!(fs::create_dir_all(&libexec_path)); + builder.copy(&path, &libexec_path.join("rust-analyzer-proc-macro-srv")); + + Some(path) + } +} + +macro_rules! tool_extended { + (($sel:ident, $builder:ident), + $($name:ident, + $path:expr, + $tool_name:expr, + stable = $stable:expr + $(,tool_std = $tool_std:literal)? + $(,allow_features = $allow_features:expr)? + $(,add_bins_to_sysroot = $add_bins_to_sysroot:expr)? + ;)+) => { + $( + #[derive(Debug, Clone, Hash, PartialEq, Eq)] + pub struct $name { + pub compiler: Compiler, + pub target: TargetSelection, + pub extra_features: Vec, + } + + impl Step for $name { + type Output = Option; + const DEFAULT: bool = true; // Overwritten below + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path($path).default_condition( + builder.config.extended + && builder.config.tools.as_ref().map_or( + // By default, on nightly/dev enable all tools, else only + // build stable tools. + $stable || builder.build.unstable_features(), + // If `tools` is set, search list for this tool. + |tools| { + tools.iter().any(|tool| match tool.as_ref() { + "clippy" => $tool_name == "clippy-driver", + x => $tool_name == x, + }) + }), + ) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure($name { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + extra_features: Vec::new(), + }); + } + + #[allow(unused_mut)] + fn run(mut $sel, $builder: &Builder<'_>) -> Option { + let tool = $builder.ensure(ToolBuild { + compiler: $sel.compiler, + target: $sel.target, + tool: $tool_name, + mode: if false $(|| $tool_std)? { Mode::ToolStd } else { Mode::ToolRustc }, + path: $path, + extra_features: $sel.extra_features, + is_optional_tool: true, + source_type: SourceType::InTree, + allow_features: concat!($($allow_features)*), + })?; + + if (false $(|| !$add_bins_to_sysroot.is_empty())?) && $sel.compiler.stage > 0 { + let bindir = $builder.sysroot($sel.compiler).join("bin"); + t!(fs::create_dir_all(&bindir)); + + #[allow(unused_variables)] + let tools_out = $builder + .cargo_out($sel.compiler, Mode::ToolRustc, $sel.target); + + $(for add_bin in $add_bins_to_sysroot { + let bin_source = tools_out.join(exe(add_bin, $sel.target)); + let bin_destination = bindir.join(exe(add_bin, $sel.compiler.host)); + $builder.copy(&bin_source, &bin_destination); + })? + + let tool = bindir.join(exe($tool_name, $sel.compiler.host)); + Some(tool) + } else { + Some(tool) + } + } + } + )+ + } +} + +// NOTE: tools need to be also added to `Builder::get_step_descriptions` in `builder.rs` +// to make `./x.py build ` work. +// NOTE: Most submodule updates for tools are handled by bootstrap.py, since they're needed just to +// invoke Cargo to build bootstrap. See the comment there for more details. +tool_extended!((self, builder), + Cargofmt, "src/tools/rustfmt", "cargo-fmt", stable=true; + CargoClippy, "src/tools/clippy", "cargo-clippy", stable=true; + Clippy, "src/tools/clippy", "clippy-driver", stable=true, add_bins_to_sysroot = ["clippy-driver", "cargo-clippy"]; + Miri, "src/tools/miri", "miri", stable=false, add_bins_to_sysroot = ["miri"]; + CargoMiri, "src/tools/miri/cargo-miri", "cargo-miri", stable=true, add_bins_to_sysroot = ["cargo-miri"]; + // FIXME: tool_std is not quite right, we shouldn't allow nightly features. + // But `builder.cargo` doesn't know how to handle ToolBootstrap in stages other than 0, + // and this is close enough for now. + Rls, "src/tools/rls", "rls", stable=true, tool_std=true; + RustDemangler, "src/tools/rust-demangler", "rust-demangler", stable=false, tool_std=true; + Rustfmt, "src/tools/rustfmt", "rustfmt", stable=true, add_bins_to_sysroot = ["rustfmt", "cargo-fmt"]; +); + +impl<'a> Builder<'a> { + /// Gets a `Command` which is ready to run `tool` in `stage` built for + /// `host`. + pub fn tool_cmd(&self, tool: Tool) -> Command { + let mut cmd = Command::new(self.tool_exe(tool)); + let compiler = self.compiler(0, self.config.build); + let host = &compiler.host; + // Prepares the `cmd` provided to be able to run the `compiler` provided. + // + // Notably this munges the dynamic library lookup path to point to the + // right location to run `compiler`. + let mut lib_paths: Vec = vec![ + self.build.rustc_snapshot_libdir(), + self.cargo_out(compiler, Mode::ToolBootstrap, *host).join("deps"), + ]; + + // On MSVC a tool may invoke a C compiler (e.g., compiletest in run-make + // mode) and that C compiler may need some extra PATH modification. Do + // so here. + if compiler.host.contains("msvc") { + let curpaths = env::var_os("PATH").unwrap_or_default(); + let curpaths = env::split_paths(&curpaths).collect::>(); + for &(ref k, ref v) in self.cc.borrow()[&compiler.host].env() { + if k != "PATH" { + continue; + } + for path in env::split_paths(v) { + if !curpaths.contains(&path) { + lib_paths.push(path); + } + } + } + } + + add_dylib_path(lib_paths, &mut cmd); + + // Provide a RUSTC for this command to use. + cmd.env("RUSTC", &self.initial_rustc); + + cmd + } +} diff --git a/src/bootstrap/src/core/build_steps/toolstate.rs b/src/bootstrap/src/core/build_steps/toolstate.rs new file mode 100644 index 000000000..a451f92b6 --- /dev/null +++ b/src/bootstrap/src/core/build_steps/toolstate.rs @@ -0,0 +1,478 @@ +use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::utils::helpers::t; +use serde_derive::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::env; +use std::fmt; +use std::fs; +use std::io::{Seek, SeekFrom}; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time; + +// Each cycle is 42 days long (6 weeks); the last week is 35..=42 then. +const BETA_WEEK_START: u64 = 35; + +#[cfg(target_os = "linux")] +const OS: Option<&str> = Some("linux"); + +#[cfg(windows)] +const OS: Option<&str> = Some("windows"); + +#[cfg(all(not(target_os = "linux"), not(windows)))] +const OS: Option<&str> = None; + +type ToolstateData = HashMap, ToolState>; + +#[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, PartialOrd)] +#[serde(rename_all = "kebab-case")] +/// Whether a tool can be compiled, tested or neither +pub enum ToolState { + /// The tool compiles successfully, but the test suite fails + TestFail = 1, + /// The tool compiles successfully and its test suite passes + TestPass = 2, + /// The tool can't even be compiled + BuildFail = 0, +} + +impl fmt::Display for ToolState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}", + match self { + ToolState::TestFail => "test-fail", + ToolState::TestPass => "test-pass", + ToolState::BuildFail => "build-fail", + } + ) + } +} + +/// Number of days after the last promotion of beta. +/// Its value is 41 on the Tuesday where "Promote master to beta (T-2)" happens. +/// The Wednesday after this has value 0. +/// We track this value to prevent regressing tools in the last week of the 6-week cycle. +fn days_since_beta_promotion() -> u64 { + let since_epoch = t!(time::SystemTime::UNIX_EPOCH.elapsed()); + (since_epoch.as_secs() / 86400 - 20) % 42 +} + +// These tools must test-pass on the beta/stable channels. +// +// On the nightly channel, their build step must be attempted, but they may not +// be able to build successfully. +static STABLE_TOOLS: &[(&str, &str)] = &[ + ("book", "src/doc/book"), + ("nomicon", "src/doc/nomicon"), + ("reference", "src/doc/reference"), + ("rust-by-example", "src/doc/rust-by-example"), + ("edition-guide", "src/doc/edition-guide"), +]; + +// These tools are permitted to not build on the beta/stable channels. +// +// We do require that we checked whether they build or not on the tools builder, +// though, as otherwise we will be unable to file an issue if they start +// failing. +static NIGHTLY_TOOLS: &[(&str, &str)] = &[ + ("embedded-book", "src/doc/embedded-book"), + // ("rustc-dev-guide", "src/doc/rustc-dev-guide"), +]; + +fn print_error(tool: &str, submodule: &str) { + eprintln!(); + eprintln!("We detected that this PR updated '{tool}', but its tests failed."); + eprintln!(); + eprintln!("If you do intend to update '{tool}', please check the error messages above and"); + eprintln!("commit another update."); + eprintln!(); + eprintln!("If you do NOT intend to update '{tool}', please ensure you did not accidentally"); + eprintln!("change the submodule at '{submodule}'. You may ask your reviewer for the"); + eprintln!("proper steps."); + crate::exit!(3); +} + +fn check_changed_files(toolstates: &HashMap, ToolState>) { + // Changed files + let output = std::process::Command::new("git") + .arg("diff") + .arg("--name-status") + .arg("HEAD") + .arg("HEAD^") + .output(); + let output = match output { + Ok(o) => o, + Err(e) => { + eprintln!("Failed to get changed files: {e:?}"); + crate::exit!(1); + } + }; + + let output = t!(String::from_utf8(output.stdout)); + + for (tool, submodule) in STABLE_TOOLS.iter().chain(NIGHTLY_TOOLS.iter()) { + let changed = output.lines().any(|l| l.starts_with('M') && l.ends_with(submodule)); + eprintln!("Verifying status of {tool}..."); + if !changed { + continue; + } + + eprintln!("This PR updated '{submodule}', verifying if status is 'test-pass'..."); + if toolstates[*tool] != ToolState::TestPass { + print_error(tool, submodule); + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct ToolStateCheck; + +impl Step for ToolStateCheck { + type Output = (); + + /// Checks tool state status. + /// + /// This is intended to be used in the `checktools.sh` script. To use + /// this, set `save-toolstates` in `config.toml` so that tool status will + /// be saved to a JSON file. Then, run `x.py test --no-fail-fast` for all + /// of the tools to populate the JSON file. After that is done, this + /// command can be run to check for any status failures, and exits with an + /// error if there are any. + /// + /// This also handles publishing the results to the `history` directory of + /// the toolstate repo + /// if the env var `TOOLSTATE_PUBLISH` is set. Note that there is a + /// *separate* step of updating the `latest.json` file and creating GitHub + /// issues and comments in `src/ci/publish_toolstate.sh`, which is only + /// performed on master. (The shell/python code is intended to be migrated + /// here eventually.) + /// + /// The rules for failure are: + /// * If the PR modifies a tool, the status must be test-pass. + /// NOTE: There is intent to change this, see + /// . + /// * All "stable" tools must be test-pass on the stable or beta branches. + /// * During beta promotion week, a PR is not allowed to "regress" a + /// stable tool. That is, the status is not allowed to get worse + /// (test-pass to test-fail or build-fail). + fn run(self, builder: &Builder<'_>) { + if builder.config.dry_run() { + return; + } + + let days_since_beta_promotion = days_since_beta_promotion(); + let in_beta_week = days_since_beta_promotion >= BETA_WEEK_START; + let is_nightly = !(builder.config.channel == "beta" || builder.config.channel == "stable"); + let toolstates = builder.toolstates(); + + let mut did_error = false; + + for (tool, _) in STABLE_TOOLS.iter().chain(NIGHTLY_TOOLS.iter()) { + if !toolstates.contains_key(*tool) { + did_error = true; + eprintln!("ERROR: Tool `{tool}` was not recorded in tool state."); + } + } + + if did_error { + crate::exit!(1); + } + + check_changed_files(&toolstates); + checkout_toolstate_repo(); + let old_toolstate = read_old_toolstate(); + + for (tool, _) in STABLE_TOOLS.iter() { + let state = toolstates[*tool]; + + if state != ToolState::TestPass { + if !is_nightly { + did_error = true; + eprintln!("ERROR: Tool `{tool}` should be test-pass but is {state}"); + } else if in_beta_week { + let old_state = old_toolstate + .iter() + .find(|ts| ts.tool == *tool) + .expect("latest.json missing tool") + .state(); + if state < old_state { + did_error = true; + eprintln!( + "ERROR: Tool `{tool}` has regressed from {old_state} to {state} during beta week." + ); + } else { + // This warning only appears in the logs, which most + // people won't read. It's mostly here for testing and + // debugging. + eprintln!( + "WARNING: Tool `{tool}` is not test-pass (is `{state}`), \ + this should be fixed before beta is branched." + ); + } + } + // `publish_toolstate.py` is responsible for updating + // `latest.json` and creating comments/issues warning people + // if there is a regression. That all happens in a separate CI + // job on the master branch once the PR has passed all tests + // on the `auto` branch. + } + } + + if did_error { + crate::exit!(1); + } + + if builder.config.channel == "nightly" && env::var_os("TOOLSTATE_PUBLISH").is_some() { + commit_toolstate_change(&toolstates); + } + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("check-tools") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(ToolStateCheck); + } +} + +impl Builder<'_> { + fn toolstates(&self) -> HashMap, ToolState> { + if let Some(ref path) = self.config.save_toolstates { + if let Some(parent) = path.parent() { + // Ensure the parent directory always exists + t!(std::fs::create_dir_all(parent)); + } + let mut file = + t!(fs::OpenOptions::new().create(true).write(true).read(true).open(path)); + + serde_json::from_reader(&mut file).unwrap_or_default() + } else { + Default::default() + } + } + + /// Updates the actual toolstate of a tool. + /// + /// The toolstates are saved to the file specified by the key + /// `rust.save-toolstates` in `config.toml`. If unspecified, nothing will be + /// done. The file is updated immediately after this function completes. + pub fn save_toolstate(&self, tool: &str, state: ToolState) { + use std::io::Write; + + // If we're in a dry run setting we don't want to save toolstates as + // that means if we e.g. panic down the line it'll look like we tested + // everything (but we actually haven't). + if self.config.dry_run() { + return; + } + // Toolstate isn't tracked for clippy or rustfmt, but since most tools do, we avoid checking + // in all the places we could save toolstate and just do so here. + if tool == "clippy-driver" || tool == "rustfmt" { + return; + } + if let Some(ref path) = self.config.save_toolstates { + if let Some(parent) = path.parent() { + // Ensure the parent directory always exists + t!(std::fs::create_dir_all(parent)); + } + let mut file = + t!(fs::OpenOptions::new().create(true).read(true).write(true).open(path)); + + let mut current_toolstates: HashMap, ToolState> = + serde_json::from_reader(&mut file).unwrap_or_default(); + current_toolstates.insert(tool.into(), state); + t!(file.seek(SeekFrom::Start(0))); + t!(file.set_len(0)); + t!(serde_json::to_writer(&file, ¤t_toolstates)); + t!(writeln!(file)); // make sure this ends in a newline + } + } +} + +fn toolstate_repo() -> String { + env::var("TOOLSTATE_REPO") + .unwrap_or_else(|_| "https://github.com/rust-lang-nursery/rust-toolstate.git".to_string()) +} + +/// Directory where the toolstate repo is checked out. +const TOOLSTATE_DIR: &str = "rust-toolstate"; + +/// Checks out the toolstate repo into `TOOLSTATE_DIR`. +fn checkout_toolstate_repo() { + if let Ok(token) = env::var("TOOLSTATE_REPO_ACCESS_TOKEN") { + prepare_toolstate_config(&token); + } + if Path::new(TOOLSTATE_DIR).exists() { + eprintln!("Cleaning old toolstate directory..."); + t!(fs::remove_dir_all(TOOLSTATE_DIR)); + } + + let status = Command::new("git") + .arg("clone") + .arg("--depth=1") + .arg(toolstate_repo()) + .arg(TOOLSTATE_DIR) + .status(); + let success = match status { + Ok(s) => s.success(), + Err(_) => false, + }; + if !success { + panic!("git clone unsuccessful (status: {status:?})"); + } +} + +/// Sets up config and authentication for modifying the toolstate repo. +fn prepare_toolstate_config(token: &str) { + fn git_config(key: &str, value: &str) { + let status = Command::new("git").arg("config").arg("--global").arg(key).arg(value).status(); + let success = match status { + Ok(s) => s.success(), + Err(_) => false, + }; + if !success { + panic!("git config key={key} value={value} failed (status: {status:?})"); + } + } + + // If changing anything here, then please check that `src/ci/publish_toolstate.sh` is up to date + // as well. + git_config("user.email", "7378925+rust-toolstate-update@users.noreply.github.com"); + git_config("user.name", "Rust Toolstate Update"); + git_config("credential.helper", "store"); + + let credential = format!("https://{token}:x-oauth-basic@github.com\n",); + let git_credential_path = PathBuf::from(t!(env::var("HOME"))).join(".git-credentials"); + t!(fs::write(&git_credential_path, credential)); +} + +/// Reads the latest toolstate from the toolstate repo. +fn read_old_toolstate() -> Vec { + let latest_path = Path::new(TOOLSTATE_DIR).join("_data").join("latest.json"); + let old_toolstate = t!(fs::read(latest_path)); + t!(serde_json::from_slice(&old_toolstate)) +} + +/// This function `commit_toolstate_change` provides functionality for pushing a change +/// to the `rust-toolstate` repository. +/// +/// The function relies on a GitHub bot user, which should have a Personal access +/// token defined in the environment variable $TOOLSTATE_REPO_ACCESS_TOKEN. If for +/// some reason you need to change the token, please update the Azure Pipelines +/// variable group. +/// +/// 1. Generate a new Personal access token: +/// +/// * Login to the bot account, and go to Settings -> Developer settings -> +/// Personal access tokens +/// * Click "Generate new token" +/// * Enable the "public_repo" permission, then click "Generate token" +/// * Copy the generated token (should be a 40-digit hexadecimal number). +/// Save it somewhere secure, as the token would be gone once you leave +/// the page. +/// +/// 2. Update the variable group in Azure Pipelines +/// +/// * Ping a member of the infrastructure team to do this. +/// +/// 4. Replace the email address below if the bot account identity is changed +/// +/// * See +/// if a private email by GitHub is wanted. +fn commit_toolstate_change(current_toolstate: &ToolstateData) { + let message = format!("({} CI update)", OS.expect("linux/windows only")); + let mut success = false; + for _ in 1..=5 { + // Upload the test results (the new commit-to-toolstate mapping) to the toolstate repo. + // This does *not* change the "current toolstate"; that only happens post-landing + // via `src/ci/docker/publish_toolstate.sh`. + publish_test_results(¤t_toolstate); + + // `git commit` failing means nothing to commit. + let status = t!(Command::new("git") + .current_dir(TOOLSTATE_DIR) + .arg("commit") + .arg("-a") + .arg("-m") + .arg(&message) + .status()); + if !status.success() { + success = true; + break; + } + + let status = t!(Command::new("git") + .current_dir(TOOLSTATE_DIR) + .arg("push") + .arg("origin") + .arg("master") + .status()); + // If we successfully push, exit. + if status.success() { + success = true; + break; + } + eprintln!("Sleeping for 3 seconds before retrying push"); + std::thread::sleep(std::time::Duration::from_secs(3)); + let status = t!(Command::new("git") + .current_dir(TOOLSTATE_DIR) + .arg("fetch") + .arg("origin") + .arg("master") + .status()); + assert!(status.success()); + let status = t!(Command::new("git") + .current_dir(TOOLSTATE_DIR) + .arg("reset") + .arg("--hard") + .arg("origin/master") + .status()); + assert!(status.success()); + } + + if !success { + panic!("Failed to update toolstate repository with new data"); + } +} + +/// Updates the "history" files with the latest results. +/// +/// These results will later be promoted to `latest.json` by the +/// `publish_toolstate.py` script if the PR passes all tests and is merged to +/// master. +fn publish_test_results(current_toolstate: &ToolstateData) { + let commit = t!(std::process::Command::new("git").arg("rev-parse").arg("HEAD").output()); + let commit = t!(String::from_utf8(commit.stdout)); + + let toolstate_serialized = t!(serde_json::to_string(¤t_toolstate)); + + let history_path = Path::new(TOOLSTATE_DIR) + .join("history") + .join(format!("{}.tsv", OS.expect("linux/windows only"))); + let mut file = t!(fs::read_to_string(&history_path)); + let end_of_first_line = file.find('\n').unwrap(); + file.insert_str(end_of_first_line, &format!("\n{}\t{}", commit.trim(), toolstate_serialized)); + t!(fs::write(&history_path, file)); +} + +#[derive(Debug, Deserialize)] +struct RepoState { + tool: String, + windows: ToolState, + linux: ToolState, +} + +impl RepoState { + fn state(&self) -> ToolState { + if cfg!(target_os = "linux") { + self.linux + } else if cfg!(windows) { + self.windows + } else { + unimplemented!() + } + } +} diff --git a/src/bootstrap/src/core/builder.rs b/src/bootstrap/src/core/builder.rs new file mode 100644 index 000000000..cd276674d --- /dev/null +++ b/src/bootstrap/src/core/builder.rs @@ -0,0 +1,2392 @@ +use std::any::{type_name, Any}; +use std::cell::{Cell, RefCell}; +use std::collections::BTreeSet; +use std::env; +use std::ffi::{OsStr, OsString}; +use std::fmt::{Debug, Write}; +use std::fs::{self, File}; +use std::hash::Hash; +use std::io::{BufRead, BufReader}; +use std::ops::Deref; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{Duration, Instant}; + +use crate::core::build_steps::llvm; +use crate::core::build_steps::tool::{self, SourceType}; +use crate::core::build_steps::{check, clean, compile, dist, doc, install, run, setup, test}; +use crate::core::config::flags::{Color, Subcommand}; +use crate::core::config::{DryRun, SplitDebuginfo, TargetSelection}; +use crate::utils::cache::{Cache, Interned, INTERNER}; +use crate::utils::helpers::{self, add_dylib_path, add_link_lib_path, exe, libdir, output, t}; +use crate::Crate; +use crate::EXTRA_CHECK_CFGS; +use crate::{Build, CLang, DocTests, GitRepo, Mode}; + +pub use crate::Compiler; +// FIXME: +// - use std::lazy for `Lazy` +// - use std::cell for `OnceCell` +// Once they get stabilized and reach beta. +use clap::ValueEnum; +use once_cell::sync::{Lazy, OnceCell}; + +#[cfg(test)] +#[path = "../tests/builder.rs"] +mod tests; + +pub struct Builder<'a> { + pub build: &'a Build, + pub top_stage: u32, + pub kind: Kind, + cache: Cache, + stack: RefCell>>, + time_spent_on_dependencies: Cell, + pub paths: Vec, +} + +impl<'a> Deref for Builder<'a> { + type Target = Build; + + fn deref(&self) -> &Self::Target { + self.build + } +} + +pub trait Step: 'static + Clone + Debug + PartialEq + Eq + Hash { + /// `PathBuf` when directories are created or to return a `Compiler` once + /// it's been assembled. + type Output: Clone; + + /// Whether this step is run by default as part of its respective phase. + /// `true` here can still be overwritten by `should_run` calling `default_condition`. + const DEFAULT: bool = false; + + /// If true, then this rule should be skipped if --target was specified, but --host was not + const ONLY_HOSTS: bool = false; + + /// Primary function to execute this rule. Can call `builder.ensure()` + /// with other steps to run those. + fn run(self, builder: &Builder<'_>) -> Self::Output; + + /// When bootstrap is passed a set of paths, this controls whether this rule + /// will execute. However, it does not get called in a "default" context + /// when we are not passed any paths; in that case, `make_run` is called + /// directly. + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_>; + + /// Builds up a "root" rule, either as a default rule or from a path passed + /// to us. + /// + /// When path is `None`, we are executing in a context where no paths were + /// passed. When `./x.py build` is run, for example, this rule could get + /// called if it is in the correct list below with a path of `None`. + fn make_run(_run: RunConfig<'_>) { + // It is reasonable to not have an implementation of make_run for rules + // who do not want to get called from the root context. This means that + // they are likely dependencies (e.g., sysroot creation) or similar, and + // as such calling them from ./x.py isn't logical. + unimplemented!() + } +} + +pub struct RunConfig<'a> { + pub builder: &'a Builder<'a>, + pub target: TargetSelection, + pub paths: Vec, +} + +impl RunConfig<'_> { + pub fn build_triple(&self) -> TargetSelection { + self.builder.build.build + } + + /// Return a list of crate names selected by `run.paths`. + #[track_caller] + pub fn cargo_crates_in_set(&self) -> Interned> { + let mut crates = Vec::new(); + for krate in &self.paths { + let path = krate.assert_single_path(); + let Some(crate_name) = self.builder.crate_paths.get(&path.path) else { + panic!("missing crate for path {}", path.path.display()) + }; + crates.push(crate_name.to_string()); + } + INTERNER.intern_list(crates) + } + + /// Given an `alias` selected by the `Step` and the paths passed on the command line, + /// return a list of the crates that should be built. + /// + /// Normally, people will pass *just* `library` if they pass it. + /// But it's possible (although strange) to pass something like `library std core`. + /// Build all crates anyway, as if they hadn't passed the other args. + pub fn make_run_crates(&self, alias: Alias) -> Interned> { + let has_alias = + self.paths.iter().any(|set| set.assert_single_path().path.ends_with(alias.as_str())); + if !has_alias { + return self.cargo_crates_in_set(); + } + + let crates = match alias { + Alias::Library => self.builder.in_tree_crates("sysroot", Some(self.target)), + Alias::Compiler => self.builder.in_tree_crates("rustc-main", Some(self.target)), + }; + + let crate_names = crates.into_iter().map(|krate| krate.name.to_string()).collect(); + INTERNER.intern_list(crate_names) + } +} + +#[derive(Debug, Copy, Clone)] +pub enum Alias { + Library, + Compiler, +} + +impl Alias { + fn as_str(self) -> &'static str { + match self { + Alias::Library => "library", + Alias::Compiler => "compiler", + } + } +} + +/// A description of the crates in this set, suitable for passing to `builder.info`. +/// +/// `crates` should be generated by [`RunConfig::cargo_crates_in_set`]. +pub fn crate_description(crates: &[impl AsRef]) -> String { + if crates.is_empty() { + return "".into(); + } + + let mut descr = String::from(" {"); + descr.push_str(crates[0].as_ref()); + for krate in &crates[1..] { + descr.push_str(", "); + descr.push_str(krate.as_ref()); + } + descr.push('}'); + descr +} + +struct StepDescription { + default: bool, + only_hosts: bool, + should_run: fn(ShouldRun<'_>) -> ShouldRun<'_>, + make_run: fn(RunConfig<'_>), + name: &'static str, + kind: Kind, +} + +#[derive(Clone, PartialOrd, Ord, PartialEq, Eq)] +pub struct TaskPath { + pub path: PathBuf, + pub kind: Option, +} + +impl Debug for TaskPath { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some(kind) = &self.kind { + write!(f, "{}::", kind.as_str())?; + } + write!(f, "{}", self.path.display()) + } +} + +/// Collection of paths used to match a task rule. +#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)] +pub enum PathSet { + /// A collection of individual paths or aliases. + /// + /// These are generally matched as a path suffix. For example, a + /// command-line value of `std` will match if `library/std` is in the + /// set. + /// + /// NOTE: the paths within a set should always be aliases of one another. + /// For example, `src/librustdoc` and `src/tools/rustdoc` should be in the same set, + /// but `library/core` and `library/std` generally should not, unless there's no way (for that Step) + /// to build them separately. + Set(BTreeSet), + /// A "suite" of paths. + /// + /// These can match as a path suffix (like `Set`), or as a prefix. For + /// example, a command-line value of `tests/ui/abi/variadic-ffi.rs` + /// will match `tests/ui`. A command-line value of `ui` would also + /// match `tests/ui`. + Suite(TaskPath), +} + +impl PathSet { + fn empty() -> PathSet { + PathSet::Set(BTreeSet::new()) + } + + fn one>(path: P, kind: Kind) -> PathSet { + let mut set = BTreeSet::new(); + set.insert(TaskPath { path: path.into(), kind: Some(kind) }); + PathSet::Set(set) + } + + fn has(&self, needle: &Path, module: Kind) -> bool { + match self { + PathSet::Set(set) => set.iter().any(|p| Self::check(p, needle, module)), + PathSet::Suite(suite) => Self::check(suite, needle, module), + } + } + + // internal use only + fn check(p: &TaskPath, needle: &Path, module: Kind) -> bool { + if let Some(p_kind) = &p.kind { + p.path.ends_with(needle) && *p_kind == module + } else { + p.path.ends_with(needle) + } + } + + /// Return all `TaskPath`s in `Self` that contain any of the `needles`, removing the + /// matched needles. + /// + /// This is used for `StepDescription::krate`, which passes all matching crates at once to + /// `Step::make_run`, rather than calling it many times with a single crate. + /// See `tests.rs` for examples. + fn intersection_removing_matches(&self, needles: &mut Vec<&Path>, module: Kind) -> PathSet { + let mut check = |p| { + for (i, n) in needles.iter().enumerate() { + let matched = Self::check(p, n, module); + if matched { + needles.remove(i); + return true; + } + } + false + }; + match self { + PathSet::Set(set) => PathSet::Set(set.iter().filter(|&p| check(p)).cloned().collect()), + PathSet::Suite(suite) => { + if check(suite) { + self.clone() + } else { + PathSet::empty() + } + } + } + } + + /// A convenience wrapper for Steps which know they have no aliases and all their sets contain only a single path. + /// + /// This can be used with [`ShouldRun::crate_or_deps`], [`ShouldRun::path`], or [`ShouldRun::alias`]. + #[track_caller] + pub fn assert_single_path(&self) -> &TaskPath { + match self { + PathSet::Set(set) => { + assert_eq!(set.len(), 1, "called assert_single_path on multiple paths"); + set.iter().next().unwrap() + } + PathSet::Suite(_) => unreachable!("called assert_single_path on a Suite path"), + } + } +} + +impl StepDescription { + fn from(kind: Kind) -> StepDescription { + StepDescription { + default: S::DEFAULT, + only_hosts: S::ONLY_HOSTS, + should_run: S::should_run, + make_run: S::make_run, + name: std::any::type_name::(), + kind, + } + } + + fn maybe_run(&self, builder: &Builder<'_>, mut pathsets: Vec) { + pathsets.retain(|set| !self.is_excluded(builder, set)); + + if pathsets.is_empty() { + return; + } + + // Determine the targets participating in this rule. + let targets = if self.only_hosts { &builder.hosts } else { &builder.targets }; + + for target in targets { + let run = RunConfig { builder, paths: pathsets.clone(), target: *target }; + (self.make_run)(run); + } + } + + fn is_excluded(&self, builder: &Builder<'_>, pathset: &PathSet) -> bool { + if builder.config.skip.iter().any(|e| pathset.has(&e, builder.kind)) { + if !matches!(builder.config.dry_run, DryRun::SelfCheck) { + println!("Skipping {pathset:?} because it is excluded"); + } + return true; + } + + if !builder.config.skip.is_empty() && !matches!(builder.config.dry_run, DryRun::SelfCheck) { + builder.verbose(&format!( + "{:?} not skipped for {:?} -- not in {:?}", + pathset, self.name, builder.config.skip + )); + } + false + } + + fn run(v: &[StepDescription], builder: &Builder<'_>, paths: &[PathBuf]) { + let should_runs = v + .iter() + .map(|desc| (desc.should_run)(ShouldRun::new(builder, desc.kind))) + .collect::>(); + + // sanity checks on rules + for (desc, should_run) in v.iter().zip(&should_runs) { + assert!( + !should_run.paths.is_empty(), + "{:?} should have at least one pathset", + desc.name + ); + } + + if paths.is_empty() || builder.config.include_default_paths { + for (desc, should_run) in v.iter().zip(&should_runs) { + if desc.default && should_run.is_really_default() { + desc.maybe_run(builder, should_run.paths.iter().cloned().collect()); + } + } + } + + // strip CurDir prefix if present + let mut paths: Vec<_> = + paths.into_iter().map(|p| p.strip_prefix(".").unwrap_or(p)).collect(); + + // Handle all test suite paths. + // (This is separate from the loop below to avoid having to handle multiple paths in `is_suite_path` somehow.) + paths.retain(|path| { + for (desc, should_run) in v.iter().zip(&should_runs) { + if let Some(suite) = should_run.is_suite_path(&path) { + desc.maybe_run(builder, vec![suite.clone()]); + return false; + } + } + true + }); + + if paths.is_empty() { + return; + } + + // Handle all PathSets. + for (desc, should_run) in v.iter().zip(&should_runs) { + let pathsets = should_run.pathset_for_paths_removing_matches(&mut paths, desc.kind); + if !pathsets.is_empty() { + desc.maybe_run(builder, pathsets); + } + } + + if !paths.is_empty() { + eprintln!("ERROR: no `{}` rules matched {:?}", builder.kind.as_str(), paths,); + eprintln!( + "HELP: run `x.py {} --help --verbose` to show a list of available paths", + builder.kind.as_str() + ); + eprintln!( + "NOTE: if you are adding a new Step to bootstrap itself, make sure you register it with `describe!`" + ); + crate::exit!(1); + } + } +} + +enum ReallyDefault<'a> { + Bool(bool), + Lazy(Lazy bool + 'a>>), +} + +pub struct ShouldRun<'a> { + pub builder: &'a Builder<'a>, + kind: Kind, + + // use a BTreeSet to maintain sort order + paths: BTreeSet, + + // If this is a default rule, this is an additional constraint placed on + // its run. Generally something like compiler docs being enabled. + is_really_default: ReallyDefault<'a>, +} + +impl<'a> ShouldRun<'a> { + fn new(builder: &'a Builder<'_>, kind: Kind) -> ShouldRun<'a> { + ShouldRun { + builder, + kind, + paths: BTreeSet::new(), + is_really_default: ReallyDefault::Bool(true), // by default no additional conditions + } + } + + pub fn default_condition(mut self, cond: bool) -> Self { + self.is_really_default = ReallyDefault::Bool(cond); + self + } + + pub fn lazy_default_condition(mut self, lazy_cond: Box bool + 'a>) -> Self { + self.is_really_default = ReallyDefault::Lazy(Lazy::new(lazy_cond)); + self + } + + pub fn is_really_default(&self) -> bool { + match &self.is_really_default { + ReallyDefault::Bool(val) => *val, + ReallyDefault::Lazy(lazy) => *lazy.deref(), + } + } + + /// Indicates it should run if the command-line selects the given crate or + /// any of its (local) dependencies. + /// + /// `make_run` will be called a single time with all matching command-line paths. + pub fn crate_or_deps(self, name: &str) -> Self { + let crates = self.builder.in_tree_crates(name, None); + self.crates(crates) + } + + /// Indicates it should run if the command-line selects any of the given crates. + /// + /// `make_run` will be called a single time with all matching command-line paths. + /// + /// Prefer [`ShouldRun::crate_or_deps`] to this function where possible. + pub(crate) fn crates(mut self, crates: Vec<&Crate>) -> Self { + for krate in crates { + let path = krate.local_path(self.builder); + self.paths.insert(PathSet::one(path, self.kind)); + } + self + } + + // single alias, which does not correspond to any on-disk path + pub fn alias(mut self, alias: &str) -> Self { + // exceptional case for `Kind::Setup` because its `library` + // and `compiler` options would otherwise naively match with + // `compiler` and `library` folders respectively. + assert!( + self.kind == Kind::Setup || !self.builder.src.join(alias).exists(), + "use `builder.path()` for real paths: {alias}" + ); + self.paths.insert(PathSet::Set( + std::iter::once(TaskPath { path: alias.into(), kind: Some(self.kind) }).collect(), + )); + self + } + + // single, non-aliased path + pub fn path(self, path: &str) -> Self { + self.paths(&[path]) + } + + /// Multiple aliases for the same job. + /// + /// This differs from [`path`] in that multiple calls to path will end up calling `make_run` + /// multiple times, whereas a single call to `paths` will only ever generate a single call to + /// `paths`. + /// + /// This is analogous to `all_krates`, although `all_krates` is gone now. Prefer [`path`] where possible. + /// + /// [`path`]: ShouldRun::path + pub fn paths(mut self, paths: &[&str]) -> Self { + static SUBMODULES_PATHS: OnceCell> = OnceCell::new(); + + let init_submodules_paths = |src: &PathBuf| { + let file = File::open(src.join(".gitmodules")).unwrap(); + + let mut submodules_paths = vec![]; + for line in BufReader::new(file).lines() { + if let Ok(line) = line { + let line = line.trim(); + + if line.starts_with("path") { + let actual_path = + line.split(' ').last().expect("Couldn't get value of path"); + submodules_paths.push(actual_path.to_owned()); + } + } + } + + submodules_paths + }; + + let submodules_paths = + SUBMODULES_PATHS.get_or_init(|| init_submodules_paths(&self.builder.src)); + + self.paths.insert(PathSet::Set( + paths + .iter() + .map(|p| { + // assert only if `p` isn't submodule + if submodules_paths.iter().find(|sm_p| p.contains(*sm_p)).is_none() { + assert!( + self.builder.src.join(p).exists(), + "`should_run.paths` should correspond to real on-disk paths - use `alias` if there is no relevant path: {}", + p + ); + } + + TaskPath { path: p.into(), kind: Some(self.kind) } + }) + .collect(), + )); + self + } + + /// Handles individual files (not directories) within a test suite. + fn is_suite_path(&self, requested_path: &Path) -> Option<&PathSet> { + self.paths.iter().find(|pathset| match pathset { + PathSet::Suite(suite) => requested_path.starts_with(&suite.path), + PathSet::Set(_) => false, + }) + } + + pub fn suite_path(mut self, suite: &str) -> Self { + self.paths.insert(PathSet::Suite(TaskPath { path: suite.into(), kind: Some(self.kind) })); + self + } + + // allows being more explicit about why should_run in Step returns the value passed to it + pub fn never(mut self) -> ShouldRun<'a> { + self.paths.insert(PathSet::empty()); + self + } + + /// Given a set of requested paths, return the subset which match the Step for this `ShouldRun`, + /// removing the matches from `paths`. + /// + /// NOTE: this returns multiple PathSets to allow for the possibility of multiple units of work + /// within the same step. For example, `test::Crate` allows testing multiple crates in the same + /// cargo invocation, which are put into separate sets because they aren't aliases. + /// + /// The reason we return PathSet instead of PathBuf is to allow for aliases that mean the same thing + /// (for now, just `all_krates` and `paths`, but we may want to add an `aliases` function in the future?) + fn pathset_for_paths_removing_matches( + &self, + paths: &mut Vec<&Path>, + kind: Kind, + ) -> Vec { + let mut sets = vec![]; + for pathset in &self.paths { + let subset = pathset.intersection_removing_matches(paths, kind); + if subset != PathSet::empty() { + sets.push(subset); + } + } + sets + } +} + +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] +pub enum Kind { + #[clap(alias = "b")] + Build, + #[clap(alias = "c")] + Check, + Clippy, + Fix, + Format, + #[clap(alias = "t")] + Test, + Bench, + #[clap(alias = "d")] + Doc, + Clean, + Dist, + Install, + #[clap(alias = "r")] + Run, + Setup, + Suggest, +} + +impl Kind { + pub fn parse(string: &str) -> Option { + // these strings, including the one-letter aliases, must match the x.py help text + Some(match string { + "build" | "b" => Kind::Build, + "check" | "c" => Kind::Check, + "clippy" => Kind::Clippy, + "fix" => Kind::Fix, + "fmt" => Kind::Format, + "test" | "t" => Kind::Test, + "bench" => Kind::Bench, + "doc" | "d" => Kind::Doc, + "clean" => Kind::Clean, + "dist" => Kind::Dist, + "install" => Kind::Install, + "run" | "r" => Kind::Run, + "setup" => Kind::Setup, + "suggest" => Kind::Suggest, + _ => return None, + }) + } + + pub fn as_str(&self) -> &'static str { + match self { + Kind::Build => "build", + Kind::Check => "check", + Kind::Clippy => "clippy", + Kind::Fix => "fix", + Kind::Format => "fmt", + Kind::Test => "test", + Kind::Bench => "bench", + Kind::Doc => "doc", + Kind::Clean => "clean", + Kind::Dist => "dist", + Kind::Install => "install", + Kind::Run => "run", + Kind::Setup => "setup", + Kind::Suggest => "suggest", + } + } + + pub fn description(&self) -> String { + match self { + Kind::Test => "Testing", + Kind::Bench => "Benchmarking", + Kind::Doc => "Documenting", + Kind::Run => "Running", + Kind::Suggest => "Suggesting", + _ => { + let title_letter = self.as_str()[0..1].to_ascii_uppercase(); + return format!("{title_letter}{}ing", &self.as_str()[1..]); + } + } + .to_owned() + } +} + +impl<'a> Builder<'a> { + fn get_step_descriptions(kind: Kind) -> Vec { + macro_rules! describe { + ($($rule:ty),+ $(,)?) => {{ + vec![$(StepDescription::from::<$rule>(kind)),+] + }}; + } + match kind { + Kind::Build => describe!( + compile::Std, + compile::Rustc, + compile::Assemble, + compile::CodegenBackend, + compile::StartupObjects, + tool::BuildManifest, + tool::Rustbook, + tool::ErrorIndex, + tool::UnstableBookGen, + tool::Tidy, + tool::Linkchecker, + tool::CargoTest, + tool::Compiletest, + tool::RemoteTestServer, + tool::RemoteTestClient, + tool::RustInstaller, + tool::Cargo, + tool::Rls, + tool::RustAnalyzer, + tool::RustAnalyzerProcMacroSrv, + tool::RustDemangler, + tool::Rustdoc, + tool::Clippy, + tool::CargoClippy, + llvm::Llvm, + llvm::Sanitizers, + tool::Rustfmt, + tool::Miri, + tool::CargoMiri, + llvm::Lld, + llvm::CrtBeginEnd, + tool::RustdocGUITest, + tool::OptimizedDist, + tool::CoverageDump, + ), + Kind::Check | Kind::Clippy | Kind::Fix => describe!( + check::Std, + check::Rustc, + check::Rustdoc, + check::CodegenBackend, + check::Clippy, + check::Miri, + check::CargoMiri, + check::MiroptTestTools, + check::Rls, + check::Rustfmt, + check::RustAnalyzer, + check::Bootstrap + ), + Kind::Test => describe!( + crate::core::build_steps::toolstate::ToolStateCheck, + test::ExpandYamlAnchors, + test::Tidy, + test::Ui, + test::RunPassValgrind, + test::Coverage, + test::CoverageMap, + test::CoverageRun, + test::MirOpt, + test::Codegen, + test::CodegenUnits, + test::Assembly, + test::Incremental, + test::Debuginfo, + test::UiFullDeps, + test::CodegenCranelift, + test::CodegenGCC, + test::Rustdoc, + test::CoverageRunRustdoc, + test::Pretty, + test::Crate, + test::CrateLibrustc, + test::CrateRustdoc, + test::CrateRustdocJsonTypes, + test::CrateBootstrap, + test::Linkcheck, + test::TierCheck, + test::Cargotest, + test::Cargo, + test::RustAnalyzer, + test::ErrorIndex, + test::Distcheck, + test::RunMakeFullDeps, + test::Nomicon, + test::Reference, + test::RustdocBook, + test::RustByExample, + test::TheBook, + test::UnstableBook, + test::RustcBook, + test::LintDocs, + test::RustcGuide, + test::EmbeddedBook, + test::EditionGuide, + test::Rustfmt, + test::Miri, + test::Clippy, + test::RustDemangler, + test::CompiletestTest, + test::RustdocJSStd, + test::RustdocJSNotStd, + test::RustdocGUI, + test::RustdocTheme, + test::RustdocUi, + test::RustdocJson, + test::HtmlCheck, + test::RustInstaller, + // Run bootstrap close to the end as it's unlikely to fail + test::Bootstrap, + // Run run-make last, since these won't pass without make on Windows + test::RunMake, + ), + Kind::Bench => describe!(test::Crate, test::CrateLibrustc), + Kind::Doc => describe!( + doc::UnstableBook, + doc::UnstableBookGen, + doc::TheBook, + doc::Standalone, + doc::Std, + doc::Rustc, + doc::Rustdoc, + doc::Rustfmt, + doc::ErrorIndex, + doc::Nomicon, + doc::Reference, + doc::RustdocBook, + doc::RustByExample, + doc::RustcBook, + doc::Cargo, + doc::CargoBook, + doc::Clippy, + doc::ClippyBook, + doc::Miri, + doc::EmbeddedBook, + doc::EditionGuide, + doc::StyleGuide, + doc::Tidy, + doc::Bootstrap, + ), + Kind::Dist => describe!( + dist::Docs, + dist::RustcDocs, + dist::JsonDocs, + dist::Mingw, + dist::Rustc, + dist::CodegenBackend, + dist::Std, + dist::RustcDev, + dist::Analysis, + dist::Src, + dist::Cargo, + dist::Rls, + dist::RustAnalyzer, + dist::Rustfmt, + dist::RustDemangler, + dist::Clippy, + dist::Miri, + dist::LlvmTools, + dist::RustDev, + dist::Bootstrap, + dist::Extended, + // It seems that PlainSourceTarball somehow changes how some of the tools + // perceive their dependencies (see #93033) which would invalidate fingerprints + // and force us to rebuild tools after vendoring dependencies. + // To work around this, create the Tarball after building all the tools. + dist::PlainSourceTarball, + dist::BuildManifest, + dist::ReproducibleArtifacts, + ), + Kind::Install => describe!( + install::Docs, + install::Std, + install::Cargo, + install::RustAnalyzer, + install::Rustfmt, + install::RustDemangler, + install::Clippy, + install::Miri, + install::LlvmTools, + install::Src, + install::Rustc + ), + Kind::Run => describe!( + run::ExpandYamlAnchors, + run::BuildManifest, + run::BumpStage0, + run::ReplaceVersionPlaceholder, + run::Miri, + run::CollectLicenseMetadata, + run::GenerateCopyright, + run::GenerateWindowsSys, + run::GenerateCompletions, + ), + Kind::Setup => describe!(setup::Profile, setup::Hook, setup::Link, setup::Vscode), + Kind::Clean => describe!(clean::CleanAll, clean::Rustc, clean::Std), + // special-cased in Build::build() + Kind::Format | Kind::Suggest => vec![], + } + } + + pub fn get_help(build: &Build, kind: Kind) -> Option { + let step_descriptions = Builder::get_step_descriptions(kind); + if step_descriptions.is_empty() { + return None; + } + + let builder = Self::new_internal(build, kind, vec![]); + let builder = &builder; + // The "build" kind here is just a placeholder, it will be replaced with something else in + // the following statement. + let mut should_run = ShouldRun::new(builder, Kind::Build); + for desc in step_descriptions { + should_run.kind = desc.kind; + should_run = (desc.should_run)(should_run); + } + let mut help = String::from("Available paths:\n"); + let mut add_path = |path: &Path| { + t!(write!(help, " ./x.py {} {}\n", kind.as_str(), path.display())); + }; + for pathset in should_run.paths { + match pathset { + PathSet::Set(set) => { + for path in set { + add_path(&path.path); + } + } + PathSet::Suite(path) => { + add_path(&path.path.join("...")); + } + } + } + Some(help) + } + + fn new_internal(build: &Build, kind: Kind, paths: Vec) -> Builder<'_> { + Builder { + build, + top_stage: build.config.stage, + kind, + cache: Cache::new(), + stack: RefCell::new(Vec::new()), + time_spent_on_dependencies: Cell::new(Duration::new(0, 0)), + paths, + } + } + + pub fn new(build: &Build) -> Builder<'_> { + let paths = &build.config.paths; + let (kind, paths) = match build.config.cmd { + Subcommand::Build => (Kind::Build, &paths[..]), + Subcommand::Check { .. } => (Kind::Check, &paths[..]), + Subcommand::Clippy { .. } => (Kind::Clippy, &paths[..]), + Subcommand::Fix => (Kind::Fix, &paths[..]), + Subcommand::Doc { .. } => (Kind::Doc, &paths[..]), + Subcommand::Test { .. } => (Kind::Test, &paths[..]), + Subcommand::Bench { .. } => (Kind::Bench, &paths[..]), + Subcommand::Dist => (Kind::Dist, &paths[..]), + Subcommand::Install => (Kind::Install, &paths[..]), + Subcommand::Run { .. } => (Kind::Run, &paths[..]), + Subcommand::Clean { .. } => (Kind::Clean, &paths[..]), + Subcommand::Format { .. } => (Kind::Format, &[][..]), + Subcommand::Suggest { .. } => (Kind::Suggest, &[][..]), + Subcommand::Setup { profile: ref path } => ( + Kind::Setup, + path.as_ref().map_or([].as_slice(), |path| std::slice::from_ref(path)), + ), + }; + + Self::new_internal(build, kind, paths.to_owned()) + } + + pub fn execute_cli(&self) { + self.run_step_descriptions(&Builder::get_step_descriptions(self.kind), &self.paths); + } + + pub fn default_doc(&self, paths: &[PathBuf]) { + self.run_step_descriptions(&Builder::get_step_descriptions(Kind::Doc), paths); + } + + pub fn doc_rust_lang_org_channel(&self) -> String { + let channel = match &*self.config.channel { + "stable" => &self.version, + "beta" => "beta", + "nightly" | "dev" => "nightly", + // custom build of rustdoc maybe? link to the latest stable docs just in case + _ => "stable", + }; + "https://doc.rust-lang.org/".to_owned() + channel + } + + fn run_step_descriptions(&self, v: &[StepDescription], paths: &[PathBuf]) { + StepDescription::run(v, self, paths); + } + + /// Obtain a compiler at a given stage and for a given host. Explicitly does + /// not take `Compiler` since all `Compiler` instances are meant to be + /// obtained through this function, since it ensures that they are valid + /// (i.e., built and assembled). + pub fn compiler(&self, stage: u32, host: TargetSelection) -> Compiler { + self.ensure(compile::Assemble { target_compiler: Compiler { stage, host } }) + } + + /// Similar to `compiler`, except handles the full-bootstrap option to + /// silently use the stage1 compiler instead of a stage2 compiler if one is + /// requested. + /// + /// Note that this does *not* have the side effect of creating + /// `compiler(stage, host)`, unlike `compiler` above which does have such + /// a side effect. The returned compiler here can only be used to compile + /// new artifacts, it can't be used to rely on the presence of a particular + /// sysroot. + /// + /// See `force_use_stage1` and `force_use_stage2` for documentation on what each argument is. + pub fn compiler_for( + &self, + stage: u32, + host: TargetSelection, + target: TargetSelection, + ) -> Compiler { + if self.build.force_use_stage2(stage) { + self.compiler(2, self.config.build) + } else if self.build.force_use_stage1(stage, target) { + self.compiler(1, self.config.build) + } else { + self.compiler(stage, host) + } + } + + pub fn sysroot(&self, compiler: Compiler) -> Interned { + self.ensure(compile::Sysroot::new(compiler)) + } + + /// Returns the libdir where the standard library and other artifacts are + /// found for a compiler's sysroot. + pub fn sysroot_libdir(&self, compiler: Compiler, target: TargetSelection) -> Interned { + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] + struct Libdir { + compiler: Compiler, + target: TargetSelection, + } + impl Step for Libdir { + type Output = Interned; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + fn run(self, builder: &Builder<'_>) -> Interned { + let lib = builder.sysroot_libdir_relative(self.compiler); + let sysroot = builder + .sysroot(self.compiler) + .join(lib) + .join("rustlib") + .join(self.target.triple) + .join("lib"); + // Avoid deleting the rustlib/ directory we just copied + // (in `impl Step for Sysroot`). + if !builder.download_rustc() { + builder.verbose(&format!( + "Removing sysroot {} to avoid caching bugs", + sysroot.display() + )); + let _ = fs::remove_dir_all(&sysroot); + t!(fs::create_dir_all(&sysroot)); + } + + if self.compiler.stage == 0 { + // The stage 0 compiler for the build triple is always pre-built. + // Ensure that `libLLVM.so` ends up in the target libdir, so that ui-fulldeps tests can use it when run. + dist::maybe_install_llvm_target( + builder, + self.compiler.host, + &builder.sysroot(self.compiler), + ); + } + + INTERNER.intern_path(sysroot) + } + } + self.ensure(Libdir { compiler, target }) + } + + pub fn sysroot_codegen_backends(&self, compiler: Compiler) -> PathBuf { + self.sysroot_libdir(compiler, compiler.host).with_file_name("codegen-backends") + } + + /// Returns the compiler's libdir where it stores the dynamic libraries that + /// it itself links against. + /// + /// For example this returns `/lib` on Unix and `/bin` on + /// Windows. + pub fn rustc_libdir(&self, compiler: Compiler) -> PathBuf { + if compiler.is_snapshot(self) { + self.rustc_snapshot_libdir() + } else { + match self.config.libdir_relative() { + Some(relative_libdir) if compiler.stage >= 1 => { + self.sysroot(compiler).join(relative_libdir) + } + _ => self.sysroot(compiler).join(libdir(compiler.host)), + } + } + } + + /// Returns the compiler's relative libdir where it stores the dynamic libraries that + /// it itself links against. + /// + /// For example this returns `lib` on Unix and `bin` on + /// Windows. + pub fn libdir_relative(&self, compiler: Compiler) -> &Path { + if compiler.is_snapshot(self) { + libdir(self.config.build).as_ref() + } else { + match self.config.libdir_relative() { + Some(relative_libdir) if compiler.stage >= 1 => relative_libdir, + _ => libdir(compiler.host).as_ref(), + } + } + } + + /// Returns the compiler's relative libdir where the standard library and other artifacts are + /// found for a compiler's sysroot. + /// + /// For example this returns `lib` on Unix and Windows. + pub fn sysroot_libdir_relative(&self, compiler: Compiler) -> &Path { + match self.config.libdir_relative() { + Some(relative_libdir) if compiler.stage >= 1 => relative_libdir, + _ if compiler.stage == 0 => &self.build.initial_libdir, + _ => Path::new("lib"), + } + } + + pub fn rustc_lib_paths(&self, compiler: Compiler) -> Vec { + let mut dylib_dirs = vec![self.rustc_libdir(compiler)]; + + // Ensure that the downloaded LLVM libraries can be found. + if self.config.llvm_from_ci { + let ci_llvm_lib = self.out.join(&*compiler.host.triple).join("ci-llvm").join("lib"); + dylib_dirs.push(ci_llvm_lib); + } + + dylib_dirs + } + + /// Adds the compiler's directory of dynamic libraries to `cmd`'s dynamic + /// library lookup path. + pub fn add_rustc_lib_path(&self, compiler: Compiler, cmd: &mut Command) { + // Windows doesn't need dylib path munging because the dlls for the + // compiler live next to the compiler and the system will find them + // automatically. + if cfg!(windows) { + return; + } + + add_dylib_path(self.rustc_lib_paths(compiler), cmd); + } + + /// Gets a path to the compiler specified. + pub fn rustc(&self, compiler: Compiler) -> PathBuf { + if compiler.is_snapshot(self) { + self.initial_rustc.clone() + } else { + self.sysroot(compiler).join("bin").join(exe("rustc", compiler.host)) + } + } + + /// Gets the paths to all of the compiler's codegen backends. + fn codegen_backends(&self, compiler: Compiler) -> impl Iterator { + fs::read_dir(self.sysroot_codegen_backends(compiler)) + .into_iter() + .flatten() + .filter_map(Result::ok) + .map(|entry| entry.path()) + } + + pub fn rustdoc(&self, compiler: Compiler) -> PathBuf { + self.ensure(tool::Rustdoc { compiler }) + } + + pub fn rustdoc_cmd(&self, compiler: Compiler) -> Command { + let mut cmd = Command::new(&self.bootstrap_out.join("rustdoc")); + cmd.env("RUSTC_STAGE", compiler.stage.to_string()) + .env("RUSTC_SYSROOT", self.sysroot(compiler)) + // Note that this is *not* the sysroot_libdir because rustdoc must be linked + // equivalently to rustc. + .env("RUSTDOC_LIBDIR", self.rustc_libdir(compiler)) + .env("CFG_RELEASE_CHANNEL", &self.config.channel) + .env("RUSTDOC_REAL", self.rustdoc(compiler)) + .env("RUSTC_BOOTSTRAP", "1"); + + cmd.arg("-Wrustdoc::invalid_codeblock_attributes"); + + if self.config.deny_warnings { + cmd.arg("-Dwarnings"); + } + cmd.arg("-Znormalize-docs"); + + // Remove make-related flags that can cause jobserver problems. + cmd.env_remove("MAKEFLAGS"); + cmd.env_remove("MFLAGS"); + + if let Some(linker) = self.linker(compiler.host) { + cmd.env("RUSTDOC_LINKER", linker); + } + cmd + } + + /// Return the path to `llvm-config` for the target, if it exists. + /// + /// Note that this returns `None` if LLVM is disabled, or if we're in a + /// check build or dry-run, where there's no need to build all of LLVM. + fn llvm_config(&self, target: TargetSelection) -> Option { + if self.config.llvm_enabled() && self.kind != Kind::Check && !self.config.dry_run() { + let llvm::LlvmResult { llvm_config, .. } = self.ensure(llvm::Llvm { target }); + if llvm_config.is_file() { + return Some(llvm_config); + } + } + None + } + + /// Like `cargo`, but only passes flags that are valid for all commands. + pub fn bare_cargo( + &self, + compiler: Compiler, + mode: Mode, + target: TargetSelection, + cmd: &str, + ) -> Command { + let mut cargo = Command::new(&self.initial_cargo); + // Run cargo from the source root so it can find .cargo/config. + // This matters when using vendoring and the working directory is outside the repository. + cargo.current_dir(&self.src); + + let out_dir = self.stage_out(compiler, mode); + cargo.env("CARGO_TARGET_DIR", &out_dir).arg(cmd); + + // Found with `rg "init_env_logger\("`. If anyone uses `init_env_logger` + // from out of tree it shouldn't matter, since x.py is only used for + // building in-tree. + let color_logs = ["RUSTDOC_LOG_COLOR", "RUSTC_LOG_COLOR", "RUST_LOG_COLOR"]; + match self.build.config.color { + Color::Always => { + cargo.arg("--color=always"); + for log in &color_logs { + cargo.env(log, "always"); + } + } + Color::Never => { + cargo.arg("--color=never"); + for log in &color_logs { + cargo.env(log, "never"); + } + } + Color::Auto => {} // nothing to do + } + + if cmd != "install" { + cargo.arg("--target").arg(target.rustc_target_arg()); + } else { + assert_eq!(target, compiler.host); + } + + if self.config.rust_optimize.is_release() { + // FIXME: cargo bench/install do not accept `--release` + if cmd != "bench" && cmd != "install" { + cargo.arg("--release"); + } + } + + // Remove make-related flags to ensure Cargo can correctly set things up + cargo.env_remove("MAKEFLAGS"); + cargo.env_remove("MFLAGS"); + + cargo + } + + /// Prepares an invocation of `cargo` to be run. + /// + /// This will create a `Command` that represents a pending execution of + /// Cargo. This cargo will be configured to use `compiler` as the actual + /// rustc compiler, its output will be scoped by `mode`'s output directory, + /// it will pass the `--target` flag for the specified `target`, and will be + /// executing the Cargo command `cmd`. + pub fn cargo( + &self, + compiler: Compiler, + mode: Mode, + source_type: SourceType, + target: TargetSelection, + cmd: &str, + ) -> Cargo { + let mut cargo = self.bare_cargo(compiler, mode, target, cmd); + let out_dir = self.stage_out(compiler, mode); + + let mut hostflags = HostFlags::default(); + + // Codegen backends are not yet tracked by -Zbinary-dep-depinfo, + // so we need to explicitly clear out if they've been updated. + for backend in self.codegen_backends(compiler) { + self.clear_if_dirty(&out_dir, &backend); + } + + if cmd == "doc" || cmd == "rustdoc" { + let my_out = match mode { + // This is the intended out directory for compiler documentation. + Mode::Rustc | Mode::ToolRustc => self.compiler_doc_out(target), + Mode::Std => { + if self.config.cmd.json() { + out_dir.join(target.triple).join("json-doc") + } else { + out_dir.join(target.triple).join("doc") + } + } + _ => panic!("doc mode {mode:?} not expected"), + }; + let rustdoc = self.rustdoc(compiler); + self.clear_if_dirty(&my_out, &rustdoc); + } + + let profile_var = |name: &str| { + let profile = if self.config.rust_optimize.is_release() { "RELEASE" } else { "DEV" }; + format!("CARGO_PROFILE_{}_{}", profile, name) + }; + + // See comment in rustc_llvm/build.rs for why this is necessary, largely llvm-config + // needs to not accidentally link to libLLVM in stage0/lib. + cargo.env("REAL_LIBRARY_PATH_VAR", &helpers::dylib_path_var()); + if let Some(e) = env::var_os(helpers::dylib_path_var()) { + cargo.env("REAL_LIBRARY_PATH", e); + } + + // Set a flag for `check`/`clippy`/`fix`, so that certain build + // scripts can do less work (i.e. not building/requiring LLVM). + if cmd == "check" || cmd == "clippy" || cmd == "fix" { + // If we've not yet built LLVM, or it's stale, then bust + // the rustc_llvm cache. That will always work, even though it + // may mean that on the next non-check build we'll need to rebuild + // rustc_llvm. But if LLVM is stale, that'll be a tiny amount + // of work comparatively, and we'd likely need to rebuild it anyway, + // so that's okay. + if crate::core::build_steps::llvm::prebuilt_llvm_config(self, target).is_err() { + cargo.env("RUST_CHECK", "1"); + } + } + + let stage = if compiler.stage == 0 && self.local_rebuild { + // Assume the local-rebuild rustc already has stage1 features. + 1 + } else { + compiler.stage + }; + + let mut rustflags = Rustflags::new(target); + if stage != 0 { + if let Ok(s) = env::var("CARGOFLAGS_NOT_BOOTSTRAP") { + cargo.args(s.split_whitespace()); + } + rustflags.env("RUSTFLAGS_NOT_BOOTSTRAP"); + } else { + if let Ok(s) = env::var("CARGOFLAGS_BOOTSTRAP") { + cargo.args(s.split_whitespace()); + } + rustflags.env("RUSTFLAGS_BOOTSTRAP"); + if cmd == "clippy" { + // clippy overwrites sysroot if we pass it to cargo. + // Pass it directly to clippy instead. + // NOTE: this can't be fixed in clippy because we explicitly don't set `RUSTC`, + // so it has no way of knowing the sysroot. + rustflags.arg("--sysroot"); + rustflags.arg( + self.sysroot(compiler) + .as_os_str() + .to_str() + .expect("sysroot must be valid UTF-8"), + ); + // Only run clippy on a very limited subset of crates (in particular, not build scripts). + cargo.arg("-Zunstable-options"); + // Explicitly does *not* set `--cfg=bootstrap`, since we're using a nightly clippy. + let host_version = Command::new("rustc").arg("--version").output().map_err(|_| ()); + let output = host_version.and_then(|output| { + if output.status.success() { + Ok(output) + } else { + Err(()) + } + }).unwrap_or_else(|_| { + eprintln!( + "ERROR: `x.py clippy` requires a host `rustc` toolchain with the `clippy` component" + ); + eprintln!("HELP: try `rustup component add clippy`"); + crate::exit!(1); + }); + if !t!(std::str::from_utf8(&output.stdout)).contains("nightly") { + rustflags.arg("--cfg=bootstrap"); + } + } else { + rustflags.arg("--cfg=bootstrap"); + } + } + + let use_new_symbol_mangling = match self.config.rust_new_symbol_mangling { + Some(setting) => { + // If an explicit setting is given, use that + setting + } + None => { + if mode == Mode::Std { + // The standard library defaults to the legacy scheme + false + } else { + // The compiler and tools default to the new scheme + true + } + } + }; + + // By default, windows-rs depends on a native library that doesn't get copied into the + // sysroot. Passing this cfg enables raw-dylib support instead, which makes the native + // library unnecessary. This can be removed when windows-rs enables raw-dylib + // unconditionally. + if let Mode::Rustc | Mode::ToolRustc = mode { + rustflags.arg("--cfg=windows_raw_dylib"); + } + + if use_new_symbol_mangling { + rustflags.arg("-Csymbol-mangling-version=v0"); + } else { + rustflags.arg("-Csymbol-mangling-version=legacy"); + rustflags.arg("-Zunstable-options"); + } + + // #[cfg(bootstrap)] + let use_new_check_cfg_syntax = self.local_rebuild; + + // Enable compile-time checking of `cfg` names, values and Cargo `features`. + // + // Note: `std`, `alloc` and `core` imports some dependencies by #[path] (like + // backtrace, core_simd, std_float, ...), those dependencies have their own + // features but cargo isn't involved in the #[path] process and so cannot pass the + // complete list of features, so for that reason we don't enable checking of + // features for std crates. + if use_new_check_cfg_syntax { + cargo.arg("-Zcheck-cfg"); + if mode == Mode::Std { + rustflags.arg("--check-cfg=cfg(feature,values(any()))"); + } + } else { + cargo.arg(if mode != Mode::Std { + "-Zcheck-cfg=names,values,output,features" + } else { + "-Zcheck-cfg=names,values,output" + }); + } + + // Add extra cfg not defined in/by rustc + // + // Note: Although it would seems that "-Zunstable-options" to `rustflags` is useless as + // cargo would implicitly add it, it was discover that sometimes bootstrap only use + // `rustflags` without `cargo` making it required. + rustflags.arg("-Zunstable-options"); + for (restricted_mode, name, values) in EXTRA_CHECK_CFGS { + if *restricted_mode == None || *restricted_mode == Some(mode) { + // Creating a string of the values by concatenating each value: + // ',"tvos","watchos"' or '' (nothing) when there are no values + let values = match values { + Some(values) => values + .iter() + .map(|val| [",", "\"", val, "\""]) + .flatten() + .collect::(), + None => String::new(), + }; + if use_new_check_cfg_syntax { + let values = values.strip_prefix(",").unwrap_or(&values); // remove the first `,` + rustflags.arg(&format!("--check-cfg=cfg({name},values({values}))")); + } else { + rustflags.arg(&format!("--check-cfg=values({name}{values})")); + } + } + } + + // FIXME(rust-lang/cargo#5754) we shouldn't be using special command arguments + // to the host invocation here, but rather Cargo should know what flags to pass rustc + // itself. + if stage == 0 { + hostflags.arg("--cfg=bootstrap"); + } + // Cargo doesn't pass RUSTFLAGS to proc_macros: + // https://github.com/rust-lang/cargo/issues/4423 + // Thus, if we are on stage 0, we explicitly set `--cfg=bootstrap`. + // We also declare that the flag is expected, which we need to do to not + // get warnings about it being unexpected. + hostflags.arg("-Zunstable-options"); + if use_new_check_cfg_syntax { + hostflags.arg("--check-cfg=cfg(bootstrap)"); + } else { + hostflags.arg("--check-cfg=values(bootstrap)"); + } + + // FIXME: It might be better to use the same value for both `RUSTFLAGS` and `RUSTDOCFLAGS`, + // but this breaks CI. At the very least, stage0 `rustdoc` needs `--cfg bootstrap`. See + // #71458. + let mut rustdocflags = rustflags.clone(); + rustdocflags.propagate_cargo_env("RUSTDOCFLAGS"); + if stage == 0 { + rustdocflags.env("RUSTDOCFLAGS_BOOTSTRAP"); + } else { + rustdocflags.env("RUSTDOCFLAGS_NOT_BOOTSTRAP"); + } + + if let Ok(s) = env::var("CARGOFLAGS") { + cargo.args(s.split_whitespace()); + } + + match mode { + Mode::Std | Mode::ToolBootstrap | Mode::ToolStd => {} + Mode::Rustc | Mode::Codegen | Mode::ToolRustc => { + // Build proc macros both for the host and the target + if target != compiler.host && cmd != "check" { + cargo.arg("-Zdual-proc-macros"); + rustflags.arg("-Zdual-proc-macros"); + } + } + } + + // This tells Cargo (and in turn, rustc) to output more complete + // dependency information. Most importantly for rustbuild, this + // includes sysroot artifacts, like libstd, which means that we don't + // need to track those in rustbuild (an error prone process!). This + // feature is currently unstable as there may be some bugs and such, but + // it represents a big improvement in rustbuild's reliability on + // rebuilds, so we're using it here. + // + // For some additional context, see #63470 (the PR originally adding + // this), as well as #63012 which is the tracking issue for this + // feature on the rustc side. + cargo.arg("-Zbinary-dep-depinfo"); + let allow_features = match mode { + Mode::ToolBootstrap | Mode::ToolStd => { + // Restrict the allowed features so we don't depend on nightly + // accidentally. + // + // binary-dep-depinfo is used by rustbuild itself for all + // compilations. + // + // Lots of tools depend on proc_macro2 and proc-macro-error. + // Those have build scripts which assume nightly features are + // available if the `rustc` version is "nighty" or "dev". See + // bin/rustc.rs for why that is a problem. Instead of labeling + // those features for each individual tool that needs them, + // just blanket allow them here. + // + // If this is ever removed, be sure to add something else in + // its place to keep the restrictions in place (or make a way + // to unset RUSTC_BOOTSTRAP). + "binary-dep-depinfo,proc_macro_span,proc_macro_span_shrink,proc_macro_diagnostic" + .to_string() + } + Mode::Std | Mode::Rustc | Mode::Codegen | Mode::ToolRustc => String::new(), + }; + + cargo.arg("-j").arg(self.jobs().to_string()); + + // FIXME: Temporary fix for https://github.com/rust-lang/cargo/issues/3005 + // Force cargo to output binaries with disambiguating hashes in the name + let mut metadata = if compiler.stage == 0 { + // Treat stage0 like a special channel, whether it's a normal prior- + // release rustc or a local rebuild with the same version, so we + // never mix these libraries by accident. + "bootstrap".to_string() + } else { + self.config.channel.to_string() + }; + // We want to make sure that none of the dependencies between + // std/test/rustc unify with one another. This is done for weird linkage + // reasons but the gist of the problem is that if librustc, libtest, and + // libstd all depend on libc from crates.io (which they actually do) we + // want to make sure they all get distinct versions. Things get really + // weird if we try to unify all these dependencies right now, namely + // around how many times the library is linked in dynamic libraries and + // such. If rustc were a static executable or if we didn't ship dylibs + // this wouldn't be a problem, but we do, so it is. This is in general + // just here to make sure things build right. If you can remove this and + // things still build right, please do! + match mode { + Mode::Std => metadata.push_str("std"), + // When we're building rustc tools, they're built with a search path + // that contains things built during the rustc build. For example, + // bitflags is built during the rustc build, and is a dependency of + // rustdoc as well. We're building rustdoc in a different target + // directory, though, which means that Cargo will rebuild the + // dependency. When we go on to build rustdoc, we'll look for + // bitflags, and find two different copies: one built during the + // rustc step and one that we just built. This isn't always a + // problem, somehow -- not really clear why -- but we know that this + // fixes things. + Mode::ToolRustc => metadata.push_str("tool-rustc"), + // Same for codegen backends. + Mode::Codegen => metadata.push_str("codegen"), + _ => {} + } + cargo.env("__CARGO_DEFAULT_LIB_METADATA", &metadata); + + if cmd == "clippy" { + rustflags.arg("-Zforce-unstable-if-unmarked"); + } + + rustflags.arg("-Zmacro-backtrace"); + + let want_rustdoc = self.doc_tests != DocTests::No; + + // We synthetically interpret a stage0 compiler used to build tools as a + // "raw" compiler in that it's the exact snapshot we download. Normally + // the stage0 build means it uses libraries build by the stage0 + // compiler, but for tools we just use the precompiled libraries that + // we've downloaded + let use_snapshot = mode == Mode::ToolBootstrap; + assert!(!use_snapshot || stage == 0 || self.local_rebuild); + + let maybe_sysroot = self.sysroot(compiler); + let sysroot = if use_snapshot { self.rustc_snapshot_sysroot() } else { &maybe_sysroot }; + let libdir = self.rustc_libdir(compiler); + + // Clear the output directory if the real rustc we're using has changed; + // Cargo cannot detect this as it thinks rustc is bootstrap/debug/rustc. + // + // Avoid doing this during dry run as that usually means the relevant + // compiler is not yet linked/copied properly. + // + // Only clear out the directory if we're compiling std; otherwise, we + // should let Cargo take care of things for us (via depdep info) + if !self.config.dry_run() && mode == Mode::Std && cmd == "build" { + self.clear_if_dirty(&out_dir, &self.rustc(compiler)); + } + + // Customize the compiler we're running. Specify the compiler to cargo + // as our shim and then pass it some various options used to configure + // how the actual compiler itself is called. + // + // These variables are primarily all read by + // src/bootstrap/bin/{rustc.rs,rustdoc.rs} + cargo + .env("RUSTBUILD_NATIVE_DIR", self.native_dir(target)) + .env("RUSTC_REAL", self.rustc(compiler)) + .env("RUSTC_STAGE", stage.to_string()) + .env("RUSTC_SYSROOT", &sysroot) + .env("RUSTC_LIBDIR", &libdir) + .env("RUSTDOC", self.bootstrap_out.join("rustdoc")) + .env( + "RUSTDOC_REAL", + if cmd == "doc" || cmd == "rustdoc" || (cmd == "test" && want_rustdoc) { + self.rustdoc(compiler) + } else { + PathBuf::from("/path/to/nowhere/rustdoc/not/required") + }, + ) + .env("RUSTC_ERROR_METADATA_DST", self.extended_error_dir()) + .env("RUSTC_BREAK_ON_ICE", "1"); + // Clippy support is a hack and uses the default `cargo-clippy` in path. + // Don't override RUSTC so that the `cargo-clippy` in path will be run. + if cmd != "clippy" { + cargo.env("RUSTC", self.bootstrap_out.join("rustc")); + } + + // Dealing with rpath here is a little special, so let's go into some + // detail. First off, `-rpath` is a linker option on Unix platforms + // which adds to the runtime dynamic loader path when looking for + // dynamic libraries. We use this by default on Unix platforms to ensure + // that our nightlies behave the same on Windows, that is they work out + // of the box. This can be disabled by setting `rpath = false` in `[rust]` + // table of `config.toml` + // + // Ok, so the astute might be wondering "why isn't `-C rpath` used + // here?" and that is indeed a good question to ask. This codegen + // option is the compiler's current interface to generating an rpath. + // Unfortunately it doesn't quite suffice for us. The flag currently + // takes no value as an argument, so the compiler calculates what it + // should pass to the linker as `-rpath`. This unfortunately is based on + // the **compile time** directory structure which when building with + // Cargo will be very different than the runtime directory structure. + // + // All that's a really long winded way of saying that if we use + // `-Crpath` then the executables generated have the wrong rpath of + // something like `$ORIGIN/deps` when in fact the way we distribute + // rustc requires the rpath to be `$ORIGIN/../lib`. + // + // So, all in all, to set up the correct rpath we pass the linker + // argument manually via `-C link-args=-Wl,-rpath,...`. Plus isn't it + // fun to pass a flag to a tool to pass a flag to pass a flag to a tool + // to change a flag in a binary? + if self.config.rpath_enabled(target) && helpers::use_host_linker(target) { + let libdir = self.sysroot_libdir_relative(compiler).to_str().unwrap(); + let rpath = if target.contains("apple") { + // Note that we need to take one extra step on macOS to also pass + // `-Wl,-instal_name,@rpath/...` to get things to work right. To + // do that we pass a weird flag to the compiler to get it to do + // so. Note that this is definitely a hack, and we should likely + // flesh out rpath support more fully in the future. + rustflags.arg("-Zosx-rpath-install-name"); + Some(format!("-Wl,-rpath,@loader_path/../{libdir}")) + } else if !target.contains("windows") + && !target.contains("aix") + && !target.contains("xous") + { + rustflags.arg("-Clink-args=-Wl,-z,origin"); + Some(format!("-Wl,-rpath,$ORIGIN/../{libdir}")) + } else { + None + }; + if let Some(rpath) = rpath { + rustflags.arg(&format!("-Clink-args={rpath}")); + } + } + + if let Some(host_linker) = self.linker(compiler.host) { + hostflags.arg(format!("-Clinker={}", host_linker.display())); + } + if self.is_fuse_ld_lld(compiler.host) { + hostflags.arg("-Clink-args=-fuse-ld=lld"); + } + + if let Some(target_linker) = self.linker(target) { + let target = crate::envify(&target.triple); + cargo.env(&format!("CARGO_TARGET_{target}_LINKER"), target_linker); + } + if self.is_fuse_ld_lld(target) { + rustflags.arg("-Clink-args=-fuse-ld=lld"); + } + self.lld_flags(target).for_each(|flag| { + rustdocflags.arg(&flag); + }); + + if !(["build", "check", "clippy", "fix", "rustc"].contains(&cmd)) && want_rustdoc { + cargo.env("RUSTDOC_LIBDIR", self.rustc_libdir(compiler)); + } + + let debuginfo_level = match mode { + Mode::Rustc | Mode::Codegen => self.config.rust_debuginfo_level_rustc, + Mode::Std => self.config.rust_debuginfo_level_std, + Mode::ToolBootstrap | Mode::ToolStd | Mode::ToolRustc => { + self.config.rust_debuginfo_level_tools + } + }; + cargo.env(profile_var("DEBUG"), debuginfo_level.to_string()); + if let Some(opt_level) = &self.config.rust_optimize.get_opt_level() { + cargo.env(profile_var("OPT_LEVEL"), opt_level); + } + if !self.config.dry_run() && self.cc.borrow()[&target].args().iter().any(|arg| arg == "-gz") + { + rustflags.arg("-Clink-arg=-gz"); + } + cargo.env( + profile_var("DEBUG_ASSERTIONS"), + if mode == Mode::Std { + self.config.rust_debug_assertions_std.to_string() + } else { + self.config.rust_debug_assertions.to_string() + }, + ); + cargo.env( + profile_var("OVERFLOW_CHECKS"), + if mode == Mode::Std { + self.config.rust_overflow_checks_std.to_string() + } else { + self.config.rust_overflow_checks.to_string() + }, + ); + + let split_debuginfo_is_stable = target.contains("linux") + || target.contains("apple") + || (target.contains("msvc") + && self.config.rust_split_debuginfo == SplitDebuginfo::Packed) + || (target.contains("windows") + && self.config.rust_split_debuginfo == SplitDebuginfo::Off); + + if !split_debuginfo_is_stable { + rustflags.arg("-Zunstable-options"); + } + match self.config.rust_split_debuginfo { + SplitDebuginfo::Packed => rustflags.arg("-Csplit-debuginfo=packed"), + SplitDebuginfo::Unpacked => rustflags.arg("-Csplit-debuginfo=unpacked"), + SplitDebuginfo::Off => rustflags.arg("-Csplit-debuginfo=off"), + }; + + if self.config.cmd.bless() { + // Bless `expect!` tests. + cargo.env("UPDATE_EXPECT", "1"); + } + + if !mode.is_tool() { + cargo.env("RUSTC_FORCE_UNSTABLE", "1"); + } + + if let Some(x) = self.crt_static(target) { + if x { + rustflags.arg("-Ctarget-feature=+crt-static"); + } else { + rustflags.arg("-Ctarget-feature=-crt-static"); + } + } + + if let Some(x) = self.crt_static(compiler.host) { + let sign = if x { "+" } else { "-" }; + hostflags.arg(format!("-Ctarget-feature={sign}crt-static")); + } + + if let Some(map_to) = self.build.debuginfo_map_to(GitRepo::Rustc) { + let map = format!("{}={}", self.build.src.display(), map_to); + cargo.env("RUSTC_DEBUGINFO_MAP", map); + + // `rustc` needs to know the virtual `/rustc/$hash` we're mapping to, + // in order to opportunistically reverse it later. + cargo.env("CFG_VIRTUAL_RUST_SOURCE_BASE_DIR", map_to); + } + + if self.config.rust_remap_debuginfo { + // FIXME: handle vendored sources + let registry_src = t!(home::cargo_home()).join("registry").join("src"); + let mut env_var = OsString::new(); + for entry in t!(std::fs::read_dir(registry_src)) { + if !env_var.is_empty() { + env_var.push("\t"); + } + env_var.push(t!(entry).path()); + env_var.push("=/rust/deps"); + } + cargo.env("RUSTC_CARGO_REGISTRY_SRC_TO_REMAP", env_var); + } + + // Enable usage of unstable features + cargo.env("RUSTC_BOOTSTRAP", "1"); + self.add_rust_test_threads(&mut cargo); + + // Almost all of the crates that we compile as part of the bootstrap may + // have a build script, including the standard library. To compile a + // build script, however, it itself needs a standard library! This + // introduces a bit of a pickle when we're compiling the standard + // library itself. + // + // To work around this we actually end up using the snapshot compiler + // (stage0) for compiling build scripts of the standard library itself. + // The stage0 compiler is guaranteed to have a libstd available for use. + // + // For other crates, however, we know that we've already got a standard + // library up and running, so we can use the normal compiler to compile + // build scripts in that situation. + if mode == Mode::Std { + cargo + .env("RUSTC_SNAPSHOT", &self.initial_rustc) + .env("RUSTC_SNAPSHOT_LIBDIR", self.rustc_snapshot_libdir()); + } else { + cargo + .env("RUSTC_SNAPSHOT", self.rustc(compiler)) + .env("RUSTC_SNAPSHOT_LIBDIR", self.rustc_libdir(compiler)); + } + + // Tools that use compiler libraries may inherit the `-lLLVM` link + // requirement, but the `-L` library path is not propagated across + // separate Cargo projects. We can add LLVM's library path to the + // platform-specific environment variable as a workaround. + if mode == Mode::ToolRustc || mode == Mode::Codegen { + if let Some(llvm_config) = self.llvm_config(target) { + let llvm_libdir = output(Command::new(&llvm_config).arg("--libdir")); + add_link_lib_path(vec![llvm_libdir.trim().into()], &mut cargo); + } + } + + // Compile everything except libraries and proc macros with the more + // efficient initial-exec TLS model. This doesn't work with `dlopen`, + // so we can't use it by default in general, but we can use it for tools + // and our own internal libraries. + if !mode.must_support_dlopen() && !target.triple.starts_with("powerpc-") { + cargo.env("RUSTC_TLS_MODEL_INITIAL_EXEC", "1"); + } + + // Ignore incremental modes except for stage0, since we're + // not guaranteeing correctness across builds if the compiler + // is changing under your feet. + if self.config.incremental && compiler.stage == 0 { + cargo.env("CARGO_INCREMENTAL", "1"); + } else { + // Don't rely on any default setting for incr. comp. in Cargo + cargo.env("CARGO_INCREMENTAL", "0"); + } + + if let Some(ref on_fail) = self.config.on_fail { + cargo.env("RUSTC_ON_FAIL", on_fail); + } + + if self.config.print_step_timings { + cargo.env("RUSTC_PRINT_STEP_TIMINGS", "1"); + } + + if self.config.print_step_rusage { + cargo.env("RUSTC_PRINT_STEP_RUSAGE", "1"); + } + + if self.config.backtrace_on_ice { + cargo.env("RUSTC_BACKTRACE_ON_ICE", "1"); + } + + cargo.env("RUSTC_VERBOSE", self.verbosity.to_string()); + + // Downstream forks of the Rust compiler might want to use a custom libc to add support for + // targets that are not yet available upstream. Adding a patch to replace libc with a + // custom one would cause compilation errors though, because Cargo would interpret the + // custom libc as part of the workspace, and apply the check-cfg lints on it. + // + // The libc build script emits check-cfg flags only when this environment variable is set, + // so this line allows the use of custom libcs. + cargo.env("LIBC_CHECK_CFG", "1"); + + if source_type == SourceType::InTree { + let mut lint_flags = Vec::new(); + // When extending this list, add the new lints to the RUSTFLAGS of the + // build_bootstrap function of src/bootstrap/bootstrap.py as well as + // some code doesn't go through this `rustc` wrapper. + lint_flags.push("-Wrust_2018_idioms"); + lint_flags.push("-Wunused_lifetimes"); + lint_flags.push("-Wsemicolon_in_expressions_from_macros"); + + if self.config.deny_warnings { + lint_flags.push("-Dwarnings"); + rustdocflags.arg("-Dwarnings"); + } + + // This does not use RUSTFLAGS due to caching issues with Cargo. + // Clippy is treated as an "in tree" tool, but shares the same + // cache as other "submodule" tools. With these options set in + // RUSTFLAGS, that causes *every* shared dependency to be rebuilt. + // By injecting this into the rustc wrapper, this circumvents + // Cargo's fingerprint detection. This is fine because lint flags + // are always ignored in dependencies. Eventually this should be + // fixed via better support from Cargo. + cargo.env("RUSTC_LINT_FLAGS", lint_flags.join(" ")); + + rustdocflags.arg("-Wrustdoc::invalid_codeblock_attributes"); + } + + if mode == Mode::Rustc { + rustflags.arg("-Zunstable-options"); + rustflags.arg("-Wrustc::internal"); + } + + // Throughout the build Cargo can execute a number of build scripts + // compiling C/C++ code and we need to pass compilers, archivers, flags, etc + // obtained previously to those build scripts. + // Build scripts use either the `cc` crate or `configure/make` so we pass + // the options through environment variables that are fetched and understood by both. + // + // FIXME: the guard against msvc shouldn't need to be here + if target.contains("msvc") { + if let Some(ref cl) = self.config.llvm_clang_cl { + cargo.env("CC", cl).env("CXX", cl); + } + } else { + let ccache = self.config.ccache.as_ref(); + let ccacheify = |s: &Path| { + let ccache = match ccache { + Some(ref s) => s, + None => return s.display().to_string(), + }; + // FIXME: the cc-rs crate only recognizes the literal strings + // `ccache` and `sccache` when doing caching compilations, so we + // mirror that here. It should probably be fixed upstream to + // accept a new env var or otherwise work with custom ccache + // vars. + match &ccache[..] { + "ccache" | "sccache" => format!("{} {}", ccache, s.display()), + _ => s.display().to_string(), + } + }; + let triple_underscored = target.triple.replace("-", "_"); + let cc = ccacheify(&self.cc(target)); + cargo.env(format!("CC_{triple_underscored}"), &cc); + + let cflags = self.cflags(target, GitRepo::Rustc, CLang::C).join(" "); + cargo.env(format!("CFLAGS_{triple_underscored}"), &cflags); + + if let Some(ar) = self.ar(target) { + let ranlib = format!("{} s", ar.display()); + cargo + .env(format!("AR_{triple_underscored}"), ar) + .env(format!("RANLIB_{triple_underscored}"), ranlib); + } + + if let Ok(cxx) = self.cxx(target) { + let cxx = ccacheify(&cxx); + let cxxflags = self.cflags(target, GitRepo::Rustc, CLang::Cxx).join(" "); + cargo + .env(format!("CXX_{triple_underscored}"), &cxx) + .env(format!("CXXFLAGS_{triple_underscored}"), cxxflags); + } + } + + // If Control Flow Guard is enabled, pass the `control-flow-guard` flag to rustc + // when compiling the standard library, since this might be linked into the final outputs + // produced by rustc. Since this mitigation is only available on Windows, only enable it + // for the standard library in case the compiler is run on a non-Windows platform. + // This is not needed for stage 0 artifacts because these will only be used for building + // the stage 1 compiler. + if cfg!(windows) + && mode == Mode::Std + && self.config.control_flow_guard + && compiler.stage >= 1 + { + rustflags.arg("-Ccontrol-flow-guard"); + } + + // For `cargo doc` invocations, make rustdoc print the Rust version into the docs + // This replaces spaces with tabs because RUSTDOCFLAGS does not + // support arguments with regular spaces. Hopefully someday Cargo will + // have space support. + let rust_version = self.rust_version().replace(' ', "\t"); + rustdocflags.arg("--crate-version").arg(&rust_version); + + // Environment variables *required* throughout the build + // + // FIXME: should update code to not require this env var + cargo.env("CFG_COMPILER_HOST_TRIPLE", target.triple); + + // Set this for all builds to make sure doc builds also get it. + cargo.env("CFG_RELEASE_CHANNEL", &self.config.channel); + + // This one's a bit tricky. As of the time of this writing the compiler + // links to the `winapi` crate on crates.io. This crate provides raw + // bindings to Windows system functions, sort of like libc does for + // Unix. This crate also, however, provides "import libraries" for the + // MinGW targets. There's an import library per dll in the windows + // distribution which is what's linked to. These custom import libraries + // are used because the winapi crate can reference Windows functions not + // present in the MinGW import libraries. + // + // For example MinGW may ship libdbghelp.a, but it may not have + // references to all the functions in the dbghelp dll. Instead the + // custom import library for dbghelp in the winapi crates has all this + // information. + // + // Unfortunately for us though the import libraries are linked by + // default via `-ldylib=winapi_foo`. That is, they're linked with the + // `dylib` type with a `winapi_` prefix (so the winapi ones don't + // conflict with the system MinGW ones). This consequently means that + // the binaries we ship of things like rustc_codegen_llvm (aka the rustc_codegen_llvm + // DLL) when linked against *again*, for example with procedural macros + // or plugins, will trigger the propagation logic of `-ldylib`, passing + // `-lwinapi_foo` to the linker again. This isn't actually available in + // our distribution, however, so the link fails. + // + // To solve this problem we tell winapi to not use its bundled import + // libraries. This means that it will link to the system MinGW import + // libraries by default, and the `-ldylib=foo` directives will still get + // passed to the final linker, but they'll look like `-lfoo` which can + // be resolved because MinGW has the import library. The downside is we + // don't get newer functions from Windows, but we don't use any of them + // anyway. + if !mode.is_tool() { + cargo.env("WINAPI_NO_BUNDLED_LIBRARIES", "1"); + } + + for _ in 0..self.verbosity { + cargo.arg("-v"); + } + + match (mode, self.config.rust_codegen_units_std, self.config.rust_codegen_units) { + (Mode::Std, Some(n), _) | (_, _, Some(n)) => { + cargo.env(profile_var("CODEGEN_UNITS"), n.to_string()); + } + _ => { + // Don't set anything + } + } + + if self.config.locked_deps { + cargo.arg("--locked"); + } + if self.config.vendor || self.is_sudo { + cargo.arg("--frozen"); + } + + // Try to use a sysroot-relative bindir, in case it was configured absolutely. + cargo.env("RUSTC_INSTALL_BINDIR", self.config.bindir_relative()); + + self.ci_env.force_coloring_in_ci(&mut cargo); + + // When we build Rust dylibs they're all intended for intermediate + // usage, so make sure we pass the -Cprefer-dynamic flag instead of + // linking all deps statically into the dylib. + if matches!(mode, Mode::Std | Mode::Rustc) { + rustflags.arg("-Cprefer-dynamic"); + } + + // When building incrementally we default to a lower ThinLTO import limit + // (unless explicitly specified otherwise). This will produce a somewhat + // slower code but give way better compile times. + { + let limit = match self.config.rust_thin_lto_import_instr_limit { + Some(limit) => Some(limit), + None if self.config.incremental => Some(10), + _ => None, + }; + + if let Some(limit) = limit { + if stage == 0 || self.config.default_codegen_backend().unwrap_or_default() == "llvm" + { + rustflags.arg(&format!("-Cllvm-args=-import-instr-limit={limit}")); + } + } + } + + if matches!(mode, Mode::Std) { + if let Some(mir_opt_level) = self.config.rust_validate_mir_opts { + rustflags.arg("-Zvalidate-mir"); + rustflags.arg(&format!("-Zmir-opt-level={mir_opt_level}")); + } + // Always enable inlining MIR when building the standard library. + // Without this flag, MIR inlining is disabled when incremental compilation is enabled. + // That causes some mir-opt tests which inline functions from the standard library to + // break when incremental compilation is enabled. So this overrides the "no inlining + // during incremental builds" heuristic for the standard library. + rustflags.arg("-Zinline-mir"); + } + + // set rustc args passed from command line + let rustc_args = + self.config.cmd.rustc_args().iter().map(|s| s.to_string()).collect::>(); + if !rustc_args.is_empty() { + cargo.env("RUSTFLAGS", &rustc_args.join(" ")); + } + + Cargo { command: cargo, rustflags, rustdocflags, hostflags, allow_features } + } + + /// Ensure that a given step is built, returning its output. This will + /// cache the step, so it is safe (and good!) to call this as often as + /// needed to ensure that all dependencies are built. + pub fn ensure(&'a self, step: S) -> S::Output { + { + let mut stack = self.stack.borrow_mut(); + for stack_step in stack.iter() { + // should skip + if stack_step.downcast_ref::().map_or(true, |stack_step| *stack_step != step) { + continue; + } + let mut out = String::new(); + out += &format!("\n\nCycle in build detected when adding {step:?}\n"); + for el in stack.iter().rev() { + out += &format!("\t{el:?}\n"); + } + panic!("{}", out); + } + if let Some(out) = self.cache.get(&step) { + self.verbose_than(1, &format!("{}c {:?}", " ".repeat(stack.len()), step)); + + return out; + } + self.verbose_than(1, &format!("{}> {:?}", " ".repeat(stack.len()), step)); + stack.push(Box::new(step.clone())); + } + + #[cfg(feature = "build-metrics")] + self.metrics.enter_step(&step, self); + + let (out, dur) = { + let start = Instant::now(); + let zero = Duration::new(0, 0); + let parent = self.time_spent_on_dependencies.replace(zero); + let out = step.clone().run(self); + let dur = start.elapsed(); + let deps = self.time_spent_on_dependencies.replace(parent + dur); + (out, dur - deps) + }; + + if self.config.print_step_timings && !self.config.dry_run() { + let step_string = format!("{step:?}"); + let brace_index = step_string.find("{").unwrap_or(0); + let type_string = type_name::(); + println!( + "[TIMING] {} {} -- {}.{:03}", + &type_string.strip_prefix("bootstrap::").unwrap_or(type_string), + &step_string[brace_index..], + dur.as_secs(), + dur.subsec_millis() + ); + } + + #[cfg(feature = "build-metrics")] + self.metrics.exit_step(self); + + { + let mut stack = self.stack.borrow_mut(); + let cur_step = stack.pop().expect("step stack empty"); + assert_eq!(cur_step.downcast_ref(), Some(&step)); + } + self.verbose_than(1, &format!("{}< {:?}", " ".repeat(self.stack.borrow().len()), step)); + self.cache.put(step, out.clone()); + out + } + + /// Ensure that a given step is built *only if it's supposed to be built by default*, returning + /// its output. This will cache the step, so it's safe (and good!) to call this as often as + /// needed to ensure that all dependencies are build. + pub(crate) fn ensure_if_default>>( + &'a self, + step: S, + kind: Kind, + ) -> S::Output { + let desc = StepDescription::from::(kind); + let should_run = (desc.should_run)(ShouldRun::new(self, desc.kind)); + + // Avoid running steps contained in --skip + for pathset in &should_run.paths { + if desc.is_excluded(self, pathset) { + return None; + } + } + + // Only execute if it's supposed to run as default + if desc.default && should_run.is_really_default() { self.ensure(step) } else { None } + } + + /// Checks if any of the "should_run" paths is in the `Builder` paths. + pub(crate) fn was_invoked_explicitly(&'a self, kind: Kind) -> bool { + let desc = StepDescription::from::(kind); + let should_run = (desc.should_run)(ShouldRun::new(self, desc.kind)); + + for path in &self.paths { + if should_run.paths.iter().any(|s| s.has(path, desc.kind)) + && !desc.is_excluded( + self, + &PathSet::Suite(TaskPath { path: path.clone(), kind: Some(desc.kind) }), + ) + { + return true; + } + } + + false + } + + pub(crate) fn maybe_open_in_browser(&self, path: impl AsRef) { + if self.was_invoked_explicitly::(Kind::Doc) { + self.open_in_browser(path); + } + } + + pub(crate) fn open_in_browser(&self, path: impl AsRef) { + if self.config.dry_run() || !self.config.cmd.open() { + return; + } + + let path = path.as_ref(); + self.info(&format!("Opening doc {}", path.display())); + if let Err(err) = opener::open(path) { + self.info(&format!("{err}\n")); + } + } +} + +/// Represents flag values in `String` form with whitespace delimiter to pass it to the compiler later. +/// +/// `-Z crate-attr` flags will be applied recursively on the target code using the `rustc_parse::parser::Parser`. +/// See `rustc_builtin_macros::cmdline_attrs::inject` for more information. +#[derive(Debug, Clone)] +struct Rustflags(String, TargetSelection); + +impl Rustflags { + fn new(target: TargetSelection) -> Rustflags { + let mut ret = Rustflags(String::new(), target); + ret.propagate_cargo_env("RUSTFLAGS"); + ret + } + + /// By default, cargo will pick up on various variables in the environment. However, bootstrap + /// reuses those variables to pass additional flags to rustdoc, so by default they get overridden. + /// Explicitly add back any previous value in the environment. + /// + /// `prefix` is usually `RUSTFLAGS` or `RUSTDOCFLAGS`. + fn propagate_cargo_env(&mut self, prefix: &str) { + // Inherit `RUSTFLAGS` by default ... + self.env(prefix); + + // ... and also handle target-specific env RUSTFLAGS if they're configured. + let target_specific = format!("CARGO_TARGET_{}_{}", crate::envify(&self.1.triple), prefix); + self.env(&target_specific); + } + + fn env(&mut self, env: &str) { + if let Ok(s) = env::var(env) { + for part in s.split(' ') { + self.arg(part); + } + } + } + + fn arg(&mut self, arg: &str) -> &mut Self { + assert_eq!(arg.split(' ').count(), 1); + if !self.0.is_empty() { + self.0.push(' '); + } + self.0.push_str(arg); + self + } +} + +/// Flags that are passed to the `rustc` shim binary. +/// These flags will only be applied when compiling host code, i.e. when +/// `--target` is unset. +#[derive(Debug, Default)] +pub struct HostFlags { + rustc: Vec, +} + +impl HostFlags { + const SEPARATOR: &'static str = " "; + + /// Adds a host rustc flag. + fn arg>(&mut self, flag: S) { + let value = flag.into().trim().to_string(); + assert!(!value.contains(Self::SEPARATOR)); + self.rustc.push(value); + } + + /// Encodes all the flags into a single string. + fn encode(self) -> String { + self.rustc.join(Self::SEPARATOR) + } +} + +#[derive(Debug)] +pub struct Cargo { + command: Command, + rustflags: Rustflags, + rustdocflags: Rustflags, + hostflags: HostFlags, + allow_features: String, +} + +impl Cargo { + pub fn rustdocflag(&mut self, arg: &str) -> &mut Cargo { + self.rustdocflags.arg(arg); + self + } + pub fn rustflag(&mut self, arg: &str) -> &mut Cargo { + self.rustflags.arg(arg); + self + } + + pub fn arg(&mut self, arg: impl AsRef) -> &mut Cargo { + self.command.arg(arg.as_ref()); + self + } + + pub fn args(&mut self, args: I) -> &mut Cargo + where + I: IntoIterator, + S: AsRef, + { + for arg in args { + self.arg(arg.as_ref()); + } + self + } + + pub fn env(&mut self, key: impl AsRef, value: impl AsRef) -> &mut Cargo { + // These are managed through rustflag/rustdocflag interfaces. + assert_ne!(key.as_ref(), "RUSTFLAGS"); + assert_ne!(key.as_ref(), "RUSTDOCFLAGS"); + self.command.env(key.as_ref(), value.as_ref()); + self + } + + pub fn add_rustc_lib_path(&mut self, builder: &Builder<'_>, compiler: Compiler) { + builder.add_rustc_lib_path(compiler, &mut self.command); + } + + pub fn current_dir(&mut self, dir: &Path) -> &mut Cargo { + self.command.current_dir(dir); + self + } + + /// Adds nightly-only features that this invocation is allowed to use. + /// + /// By default, all nightly features are allowed. Once this is called, it + /// will be restricted to the given set. + pub fn allow_features(&mut self, features: &str) -> &mut Cargo { + if !self.allow_features.is_empty() { + self.allow_features.push(','); + } + self.allow_features.push_str(features); + self + } +} + +impl From for Command { + fn from(mut cargo: Cargo) -> Command { + let rustflags = &cargo.rustflags.0; + if !rustflags.is_empty() { + cargo.command.env("RUSTFLAGS", rustflags); + } + + let rustdocflags = &cargo.rustdocflags.0; + if !rustdocflags.is_empty() { + cargo.command.env("RUSTDOCFLAGS", rustdocflags); + } + + let encoded_hostflags = cargo.hostflags.encode(); + if !encoded_hostflags.is_empty() { + cargo.command.env("RUSTC_HOST_FLAGS", encoded_hostflags); + } + + if !cargo.allow_features.is_empty() { + cargo.command.env("RUSTC_ALLOW_FEATURES", cargo.allow_features); + } + + cargo.command + } +} diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs new file mode 100644 index 000000000..0a9175aa3 --- /dev/null +++ b/src/bootstrap/src/core/config/config.rs @@ -0,0 +1,2208 @@ +//! Serialized configuration of a build. +//! +//! This module implements parsing `config.toml` configuration files to tweak +//! how the build runs. + +#[cfg(test)] +#[path = "../../tests/config.rs"] +mod tests; + +use std::cell::{Cell, RefCell}; +use std::cmp; +use std::collections::{HashMap, HashSet}; +use std::env; +use std::fmt::{self, Display}; +use std::fs; +use std::io::IsTerminal; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::str::FromStr; + +use crate::core::build_steps::compile::CODEGEN_BACKEND_PREFIX; +use crate::core::build_steps::llvm; +use crate::core::config::flags::{Color, Flags, Warnings}; +use crate::utils::cache::{Interned, INTERNER}; +use crate::utils::channel::{self, GitInfo}; +use crate::utils::helpers::{exe, output, t}; +use build_helper::exit; +use once_cell::sync::OnceCell; +use semver::Version; +use serde::{Deserialize, Deserializer}; +use serde_derive::Deserialize; + +pub use crate::core::config::flags::Subcommand; +use build_helper::git::GitConfig; + +macro_rules! check_ci_llvm { + ($name:expr) => { + assert!( + $name.is_none(), + "setting {} is incompatible with download-ci-llvm.", + stringify!($name) + ); + }; +} + +#[derive(Clone, Default)] +pub enum DryRun { + /// This isn't a dry run. + #[default] + Disabled, + /// This is a dry run enabled by bootstrap itself, so it can verify that no work is done. + SelfCheck, + /// This is a dry run enabled by the `--dry-run` flag. + UserSelected, +} + +#[derive(Copy, Clone, Default, PartialEq, Eq)] +pub enum DebuginfoLevel { + #[default] + None, + LineTablesOnly, + Limited, + Full, +} + +// NOTE: can't derive(Deserialize) because the intermediate trip through toml::Value only +// deserializes i64, and derive() only generates visit_u64 +impl<'de> Deserialize<'de> for DebuginfoLevel { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + use serde::de::Error; + + Ok(match Deserialize::deserialize(deserializer)? { + StringOrInt::String("none") | StringOrInt::Int(0) => DebuginfoLevel::None, + StringOrInt::String("line-tables-only") => DebuginfoLevel::LineTablesOnly, + StringOrInt::String("limited") | StringOrInt::Int(1) => DebuginfoLevel::Limited, + StringOrInt::String("full") | StringOrInt::Int(2) => DebuginfoLevel::Full, + StringOrInt::Int(n) => { + let other = serde::de::Unexpected::Signed(n); + return Err(D::Error::invalid_value(other, &"expected 0, 1, or 2")); + } + StringOrInt::String(s) => { + let other = serde::de::Unexpected::Str(s); + return Err(D::Error::invalid_value( + other, + &"expected none, line-tables-only, limited, or full", + )); + } + }) + } +} + +/// Suitable for passing to `-C debuginfo` +impl Display for DebuginfoLevel { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use DebuginfoLevel::*; + f.write_str(match self { + None => "0", + LineTablesOnly => "line-tables-only", + Limited => "1", + Full => "2", + }) + } +} + +/// Global configuration for the entire build and/or bootstrap. +/// +/// This structure is parsed from `config.toml`, and some of the fields are inferred from `git` or build-time parameters. +/// +/// Note that this structure is not decoded directly into, but rather it is +/// filled out from the decoded forms of the structs below. For documentation +/// each field, see the corresponding fields in +/// `config.example.toml`. +#[derive(Default, Clone)] +pub struct Config { + pub changelog_seen: Option, // FIXME: Deprecated field. Remove it at 2024. + pub change_id: Option, + pub ccache: Option, + /// Call Build::ninja() instead of this. + pub ninja_in_file: bool, + pub verbose: usize, + pub submodules: Option, + pub compiler_docs: bool, + pub library_docs_private_items: bool, + pub docs_minification: bool, + pub docs: bool, + pub locked_deps: bool, + pub vendor: bool, + pub target_config: HashMap, + pub full_bootstrap: bool, + pub extended: bool, + pub tools: Option>, + pub sanitizers: bool, + pub profiler: bool, + pub omit_git_hash: bool, + pub skip: Vec, + pub include_default_paths: bool, + pub rustc_error_format: Option, + pub json_output: bool, + pub test_compare_mode: bool, + pub color: Color, + pub patch_binaries_for_nix: Option, + pub stage0_metadata: Stage0Metadata, + pub android_ndk: Option, + + pub stdout_is_tty: bool, + pub stderr_is_tty: bool, + + pub on_fail: Option, + pub stage: u32, + pub keep_stage: Vec, + pub keep_stage_std: Vec, + pub src: PathBuf, + /// defaults to `config.toml` + pub config: Option, + pub jobs: Option, + pub cmd: Subcommand, + pub incremental: bool, + pub dry_run: DryRun, + /// Arguments appearing after `--` to be forwarded to tools, + /// e.g. `--fix-broken` or test arguments. + pub free_args: Vec, + + /// `None` if we shouldn't download CI compiler artifacts, or the commit to download if we should. + #[cfg(not(test))] + download_rustc_commit: Option, + #[cfg(test)] + pub download_rustc_commit: Option, + + pub deny_warnings: bool, + pub backtrace_on_ice: bool, + + // llvm codegen options + pub llvm_assertions: bool, + pub llvm_tests: bool, + pub llvm_plugins: bool, + pub llvm_optimize: bool, + pub llvm_thin_lto: bool, + pub llvm_release_debuginfo: bool, + pub llvm_static_stdcpp: bool, + /// `None` if `llvm_from_ci` is true and we haven't yet downloaded llvm. + #[cfg(not(test))] + llvm_link_shared: Cell>, + #[cfg(test)] + pub llvm_link_shared: Cell>, + pub llvm_clang_cl: Option, + pub llvm_targets: Option, + pub llvm_experimental_targets: Option, + pub llvm_link_jobs: Option, + pub llvm_version_suffix: Option, + pub llvm_use_linker: Option, + pub llvm_allow_old_toolchain: bool, + pub llvm_polly: bool, + pub llvm_clang: bool, + pub llvm_enable_warnings: bool, + pub llvm_from_ci: bool, + pub llvm_build_config: HashMap, + + pub use_lld: bool, + pub lld_enabled: bool, + pub llvm_tools_enabled: bool, + + pub llvm_cflags: Option, + pub llvm_cxxflags: Option, + pub llvm_ldflags: Option, + pub llvm_use_libcxx: bool, + + // rust codegen options + pub rust_optimize: RustOptimize, + pub rust_codegen_units: Option, + pub rust_codegen_units_std: Option, + pub rust_debug_assertions: bool, + pub rust_debug_assertions_std: bool, + pub rust_overflow_checks: bool, + pub rust_overflow_checks_std: bool, + pub rust_debug_logging: bool, + pub rust_debuginfo_level_rustc: DebuginfoLevel, + pub rust_debuginfo_level_std: DebuginfoLevel, + pub rust_debuginfo_level_tools: DebuginfoLevel, + pub rust_debuginfo_level_tests: DebuginfoLevel, + pub rust_split_debuginfo: SplitDebuginfo, + pub rust_rpath: bool, + pub rustc_parallel: bool, + pub rustc_default_linker: Option, + pub rust_optimize_tests: bool, + pub rust_dist_src: bool, + pub rust_codegen_backends: Vec>, + pub rust_verify_llvm_ir: bool, + pub rust_thin_lto_import_instr_limit: Option, + pub rust_remap_debuginfo: bool, + pub rust_new_symbol_mangling: Option, + pub rust_profile_use: Option, + pub rust_profile_generate: Option, + pub rust_lto: RustcLto, + pub rust_validate_mir_opts: Option, + pub llvm_profile_use: Option, + pub llvm_profile_generate: bool, + pub llvm_libunwind_default: Option, + pub enable_bolt_settings: bool, + + pub reproducible_artifacts: Vec, + + pub build: TargetSelection, + pub hosts: Vec, + pub targets: Vec, + pub local_rebuild: bool, + pub jemalloc: bool, + pub control_flow_guard: bool, + + // dist misc + pub dist_sign_folder: Option, + pub dist_upload_addr: Option, + pub dist_compression_formats: Option>, + pub dist_compression_profile: String, + pub dist_include_mingw_linker: bool, + + // libstd features + pub backtrace: bool, // support for RUST_BACKTRACE + + // misc + pub low_priority: bool, + pub channel: String, + pub description: Option, + pub verbose_tests: bool, + pub save_toolstates: Option, + pub print_step_timings: bool, + pub print_step_rusage: bool, + pub missing_tools: bool, + + // Fallback musl-root for all targets + pub musl_root: Option, + pub prefix: Option, + pub sysconfdir: Option, + pub datadir: Option, + pub docdir: Option, + pub bindir: PathBuf, + pub libdir: Option, + pub mandir: Option, + pub codegen_tests: bool, + pub nodejs: Option, + pub npm: Option, + pub gdb: Option, + pub python: Option, + pub reuse: Option, + pub cargo_native_static: bool, + pub configure_args: Vec, + pub out: PathBuf, + pub rust_info: channel::GitInfo, + + // These are either the stage0 downloaded binaries or the locally installed ones. + pub initial_cargo: PathBuf, + pub initial_rustc: PathBuf, + + #[cfg(not(test))] + initial_rustfmt: RefCell, + #[cfg(test)] + pub initial_rustfmt: RefCell, + + pub paths: Vec, +} + +#[derive(Default, Deserialize, Clone)] +pub struct Stage0Metadata { + pub compiler: CompilerMetadata, + pub config: Stage0Config, + pub checksums_sha256: HashMap, + pub rustfmt: Option, +} +#[derive(Default, Deserialize, Clone)] +pub struct CompilerMetadata { + pub date: String, + pub version: String, +} + +#[derive(Default, Deserialize, Clone)] +pub struct Stage0Config { + pub dist_server: String, + pub artifacts_server: String, + pub artifacts_with_llvm_assertions_server: String, + pub git_merge_commit_email: String, + pub git_repository: String, + pub nightly_branch: String, +} +#[derive(Default, Deserialize, Clone)] +pub struct RustfmtMetadata { + pub date: String, + pub version: String, +} + +#[derive(Clone, Debug, Default)] +pub enum RustfmtState { + SystemToolchain(PathBuf), + Downloaded(PathBuf), + Unavailable, + #[default] + LazyEvaluated, +} + +#[derive(Debug, Default, Clone, Copy, PartialEq)] +pub enum LlvmLibunwind { + #[default] + No, + InTree, + System, +} + +impl FromStr for LlvmLibunwind { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "no" => Ok(Self::No), + "in-tree" => Ok(Self::InTree), + "system" => Ok(Self::System), + invalid => Err(format!("Invalid value '{invalid}' for rust.llvm-libunwind config.")), + } + } +} + +#[derive(Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum SplitDebuginfo { + Packed, + Unpacked, + #[default] + Off, +} + +impl std::str::FromStr for SplitDebuginfo { + type Err = (); + + fn from_str(s: &str) -> Result { + match s { + "packed" => Ok(SplitDebuginfo::Packed), + "unpacked" => Ok(SplitDebuginfo::Unpacked), + "off" => Ok(SplitDebuginfo::Off), + _ => Err(()), + } + } +} + +impl SplitDebuginfo { + /// Returns the default `-Csplit-debuginfo` value for the current target. See the comment for + /// `rust.split-debuginfo` in `config.example.toml`. + fn default_for_platform(target: &str) -> Self { + if target.contains("apple") { + SplitDebuginfo::Unpacked + } else if target.contains("windows") { + SplitDebuginfo::Packed + } else { + SplitDebuginfo::Off + } + } +} + +/// LTO mode used for compiling rustc itself. +#[derive(Default, Clone, PartialEq, Debug)] +pub enum RustcLto { + Off, + #[default] + ThinLocal, + Thin, + Fat, +} + +impl std::str::FromStr for RustcLto { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "thin-local" => Ok(RustcLto::ThinLocal), + "thin" => Ok(RustcLto::Thin), + "fat" => Ok(RustcLto::Fat), + "off" => Ok(RustcLto::Off), + _ => Err(format!("Invalid value for rustc LTO: {s}")), + } + } +} + +#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct TargetSelection { + pub triple: Interned, + file: Option>, + synthetic: bool, +} + +/// Newtype over `Vec` so we can implement custom parsing logic +#[derive(Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub struct TargetSelectionList(Vec); + +pub fn target_selection_list(s: &str) -> Result { + Ok(TargetSelectionList( + s.split(",").filter(|s| !s.is_empty()).map(TargetSelection::from_user).collect(), + )) +} + +impl TargetSelection { + pub fn from_user(selection: &str) -> Self { + let path = Path::new(selection); + + let (triple, file) = if path.exists() { + let triple = path + .file_stem() + .expect("Target specification file has no file stem") + .to_str() + .expect("Target specification file stem is not UTF-8"); + + (triple, Some(selection)) + } else { + (selection, None) + }; + + let triple = INTERNER.intern_str(triple); + let file = file.map(|f| INTERNER.intern_str(f)); + + Self { triple, file, synthetic: false } + } + + pub fn create_synthetic(triple: &str, file: &str) -> Self { + Self { + triple: INTERNER.intern_str(triple), + file: Some(INTERNER.intern_str(file)), + synthetic: true, + } + } + + pub fn rustc_target_arg(&self) -> &str { + self.file.as_ref().unwrap_or(&self.triple) + } + + pub fn contains(&self, needle: &str) -> bool { + self.triple.contains(needle) + } + + pub fn starts_with(&self, needle: &str) -> bool { + self.triple.starts_with(needle) + } + + pub fn ends_with(&self, needle: &str) -> bool { + self.triple.ends_with(needle) + } + + // See src/bootstrap/synthetic_targets.rs + pub fn is_synthetic(&self) -> bool { + self.synthetic + } +} + +impl fmt::Display for TargetSelection { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.triple)?; + if let Some(file) = self.file { + write!(f, "({file})")?; + } + Ok(()) + } +} + +impl fmt::Debug for TargetSelection { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{self}") + } +} + +impl PartialEq<&str> for TargetSelection { + fn eq(&self, other: &&str) -> bool { + self.triple == *other + } +} + +/// Per-target configuration stored in the global configuration structure. +#[derive(Default, Clone)] +pub struct Target { + /// Some(path to llvm-config) if using an external LLVM. + pub llvm_config: Option, + pub llvm_has_rust_patches: Option, + /// Some(path to FileCheck) if one was specified. + pub llvm_filecheck: Option, + pub llvm_libunwind: Option, + pub cc: Option, + pub cxx: Option, + pub ar: Option, + pub ranlib: Option, + pub default_linker: Option, + pub linker: Option, + pub sanitizers: Option, + pub profiler: Option, + pub rpath: Option, + pub crt_static: Option, + pub musl_root: Option, + pub musl_libdir: Option, + pub wasi_root: Option, + pub qemu_rootfs: Option, + pub no_std: bool, +} + +impl Target { + pub fn from_triple(triple: &str) -> Self { + let mut target: Self = Default::default(); + if triple.contains("-none") || triple.contains("nvptx") || triple.contains("switch") { + target.no_std = true; + } + target + } +} +/// Structure of the `config.toml` file that configuration is read from. +/// +/// This structure uses `Decodable` to automatically decode a TOML configuration +/// file into this format, and then this is traversed and written into the above +/// `Config` structure. +#[derive(Deserialize, Default)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub(crate) struct TomlConfig { + changelog_seen: Option, // FIXME: Deprecated field. Remove it at 2024. + change_id: Option, + build: Option, + install: Option, + llvm: Option, + rust: Option, + target: Option>, + dist: Option, + profile: Option, +} + +/// Describes how to handle conflicts in merging two [`TomlConfig`] +#[derive(Copy, Clone, Debug)] +enum ReplaceOpt { + /// Silently ignore a duplicated value + IgnoreDuplicate, + /// Override the current value, even if it's `Some` + Override, + /// Exit with an error on duplicate values + ErrorOnDuplicate, +} + +trait Merge { + fn merge(&mut self, other: Self, replace: ReplaceOpt); +} + +impl Merge for TomlConfig { + fn merge( + &mut self, + TomlConfig { + build, + install, + llvm, + rust, + dist, + target, + profile: _, + changelog_seen, + change_id, + }: Self, + replace: ReplaceOpt, + ) { + fn do_merge(x: &mut Option, y: Option, replace: ReplaceOpt) { + if let Some(new) = y { + if let Some(original) = x { + original.merge(new, replace); + } else { + *x = Some(new); + } + } + } + self.changelog_seen.merge(changelog_seen, replace); + self.change_id.merge(change_id, replace); + do_merge(&mut self.build, build, replace); + do_merge(&mut self.install, install, replace); + do_merge(&mut self.llvm, llvm, replace); + do_merge(&mut self.rust, rust, replace); + do_merge(&mut self.dist, dist, replace); + assert!(target.is_none(), "merging target-specific config is not currently supported"); + } +} + +// We are using a decl macro instead of a derive proc macro here to reduce the compile time of +// rustbuild. +macro_rules! define_config { + ($(#[$attr:meta])* struct $name:ident { + $($field:ident: Option<$field_ty:ty> = $field_key:literal,)* + }) => { + $(#[$attr])* + struct $name { + $($field: Option<$field_ty>,)* + } + + impl Merge for $name { + fn merge(&mut self, other: Self, replace: ReplaceOpt) { + $( + match replace { + ReplaceOpt::IgnoreDuplicate => { + if self.$field.is_none() { + self.$field = other.$field; + } + }, + ReplaceOpt::Override => { + if other.$field.is_some() { + self.$field = other.$field; + } + } + ReplaceOpt::ErrorOnDuplicate => { + if other.$field.is_some() { + if self.$field.is_some() { + if cfg!(test) { + panic!("overriding existing option") + } else { + eprintln!("overriding existing option: `{}`", stringify!($field)); + exit!(2); + } + } else { + self.$field = other.$field; + } + } + } + } + )* + } + } + + // The following is a trimmed version of what serde_derive generates. All parts not relevant + // for toml deserialization have been removed. This reduces the binary size and improves + // compile time of rustbuild. + impl<'de> Deserialize<'de> for $name { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct Field; + impl<'de> serde::de::Visitor<'de> for Field { + type Value = $name; + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(concat!("struct ", stringify!($name))) + } + + #[inline] + fn visit_map(self, mut map: A) -> Result + where + A: serde::de::MapAccess<'de>, + { + $(let mut $field: Option<$field_ty> = None;)* + while let Some(key) = + match serde::de::MapAccess::next_key::(&mut map) { + Ok(val) => val, + Err(err) => { + return Err(err); + } + } + { + match &*key { + $($field_key => { + if $field.is_some() { + return Err(::duplicate_field( + $field_key, + )); + } + $field = match serde::de::MapAccess::next_value::<$field_ty>( + &mut map, + ) { + Ok(val) => Some(val), + Err(err) => { + return Err(err); + } + }; + })* + key => { + return Err(serde::de::Error::unknown_field(key, FIELDS)); + } + } + } + Ok($name { $($field),* }) + } + } + const FIELDS: &'static [&'static str] = &[ + $($field_key,)* + ]; + Deserializer::deserialize_struct( + deserializer, + stringify!($name), + FIELDS, + Field, + ) + } + } + } +} + +impl Merge for Option { + fn merge(&mut self, other: Self, replace: ReplaceOpt) { + match replace { + ReplaceOpt::IgnoreDuplicate => { + if self.is_none() { + *self = other; + } + } + ReplaceOpt::Override => { + if other.is_some() { + *self = other; + } + } + ReplaceOpt::ErrorOnDuplicate => { + if other.is_some() { + if self.is_some() { + if cfg!(test) { + panic!("overriding existing option") + } else { + eprintln!("overriding existing option"); + exit!(2); + } + } else { + *self = other; + } + } + } + } + } +} + +define_config! { + /// TOML representation of various global build decisions. + #[derive(Default)] + struct Build { + build: Option = "build", + host: Option> = "host", + target: Option> = "target", + build_dir: Option = "build-dir", + cargo: Option = "cargo", + rustc: Option = "rustc", + rustfmt: Option = "rustfmt", + docs: Option = "docs", + compiler_docs: Option = "compiler-docs", + library_docs_private_items: Option = "library-docs-private-items", + docs_minification: Option = "docs-minification", + submodules: Option = "submodules", + gdb: Option = "gdb", + nodejs: Option = "nodejs", + npm: Option = "npm", + python: Option = "python", + reuse: Option = "reuse", + locked_deps: Option = "locked-deps", + vendor: Option = "vendor", + full_bootstrap: Option = "full-bootstrap", + extended: Option = "extended", + tools: Option> = "tools", + verbose: Option = "verbose", + sanitizers: Option = "sanitizers", + profiler: Option = "profiler", + cargo_native_static: Option = "cargo-native-static", + low_priority: Option = "low-priority", + configure_args: Option> = "configure-args", + local_rebuild: Option = "local-rebuild", + print_step_timings: Option = "print-step-timings", + print_step_rusage: Option = "print-step-rusage", + check_stage: Option = "check-stage", + doc_stage: Option = "doc-stage", + build_stage: Option = "build-stage", + test_stage: Option = "test-stage", + install_stage: Option = "install-stage", + dist_stage: Option = "dist-stage", + bench_stage: Option = "bench-stage", + patch_binaries_for_nix: Option = "patch-binaries-for-nix", + // NOTE: only parsed by bootstrap.py, `--feature build-metrics` enables metrics unconditionally + metrics: Option = "metrics", + android_ndk: Option = "android-ndk", + } +} + +define_config! { + /// TOML representation of various global install decisions. + struct Install { + prefix: Option = "prefix", + sysconfdir: Option = "sysconfdir", + docdir: Option = "docdir", + bindir: Option = "bindir", + libdir: Option = "libdir", + mandir: Option = "mandir", + datadir: Option = "datadir", + } +} + +define_config! { + /// TOML representation of how the LLVM build is configured. + struct Llvm { + optimize: Option = "optimize", + thin_lto: Option = "thin-lto", + release_debuginfo: Option = "release-debuginfo", + assertions: Option = "assertions", + tests: Option = "tests", + plugins: Option = "plugins", + ccache: Option = "ccache", + static_libstdcpp: Option = "static-libstdcpp", + ninja: Option = "ninja", + targets: Option = "targets", + experimental_targets: Option = "experimental-targets", + link_jobs: Option = "link-jobs", + link_shared: Option = "link-shared", + version_suffix: Option = "version-suffix", + clang_cl: Option = "clang-cl", + cflags: Option = "cflags", + cxxflags: Option = "cxxflags", + ldflags: Option = "ldflags", + use_libcxx: Option = "use-libcxx", + use_linker: Option = "use-linker", + allow_old_toolchain: Option = "allow-old-toolchain", + polly: Option = "polly", + clang: Option = "clang", + enable_warnings: Option = "enable-warnings", + download_ci_llvm: Option = "download-ci-llvm", + build_config: Option> = "build-config", + } +} + +define_config! { + struct Dist { + sign_folder: Option = "sign-folder", + gpg_password_file: Option = "gpg-password-file", + upload_addr: Option = "upload-addr", + src_tarball: Option = "src-tarball", + missing_tools: Option = "missing-tools", + compression_formats: Option> = "compression-formats", + compression_profile: Option = "compression-profile", + include_mingw_linker: Option = "include-mingw-linker", + } +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(untagged)] +pub enum StringOrBool { + String(String), + Bool(bool), +} + +impl Default for StringOrBool { + fn default() -> StringOrBool { + StringOrBool::Bool(false) + } +} + +impl StringOrBool { + fn is_string_or_true(&self) -> bool { + matches!(self, Self::String(_) | Self::Bool(true)) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RustOptimize { + String(String), + Int(u8), + Bool(bool), +} + +impl Default for RustOptimize { + fn default() -> RustOptimize { + RustOptimize::Bool(false) + } +} + +impl<'de> Deserialize<'de> for RustOptimize { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_any(OptimizeVisitor) + } +} + +struct OptimizeVisitor; + +impl<'de> serde::de::Visitor<'de> for OptimizeVisitor { + type Value = RustOptimize; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(r#"one of: 0, 1, 2, 3, "s", "z", true, false"#) + } + + fn visit_str(self, value: &str) -> Result + where + E: serde::de::Error, + { + if ["s", "z"].iter().find(|x| **x == value).is_some() { + Ok(RustOptimize::String(value.to_string())) + } else { + Err(format_optimize_error_msg(value)).map_err(serde::de::Error::custom) + } + } + + fn visit_i64(self, value: i64) -> Result + where + E: serde::de::Error, + { + if matches!(value, 0..=3) { + Ok(RustOptimize::Int(value as u8)) + } else { + Err(format_optimize_error_msg(value)).map_err(serde::de::Error::custom) + } + } + + fn visit_bool(self, value: bool) -> Result + where + E: serde::de::Error, + { + Ok(RustOptimize::Bool(value)) + } +} + +fn format_optimize_error_msg(v: impl std::fmt::Display) -> String { + format!( + r#"unrecognized option for rust optimize: "{v}", expected one of 0, 1, 2, 3, "s", "z", true, false"# + ) +} + +impl RustOptimize { + pub(crate) fn is_release(&self) -> bool { + match &self { + RustOptimize::Bool(true) | RustOptimize::String(_) => true, + RustOptimize::Int(i) => *i > 0, + RustOptimize::Bool(false) => false, + } + } + + pub(crate) fn get_opt_level(&self) -> Option { + match &self { + RustOptimize::String(s) => Some(s.clone()), + RustOptimize::Int(i) => Some(i.to_string()), + RustOptimize::Bool(_) => None, + } + } +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum StringOrInt<'a> { + String(&'a str), + Int(i64), +} +define_config! { + /// TOML representation of how the Rust build is configured. + struct Rust { + optimize: Option = "optimize", + debug: Option = "debug", + codegen_units: Option = "codegen-units", + codegen_units_std: Option = "codegen-units-std", + debug_assertions: Option = "debug-assertions", + debug_assertions_std: Option = "debug-assertions-std", + overflow_checks: Option = "overflow-checks", + overflow_checks_std: Option = "overflow-checks-std", + debug_logging: Option = "debug-logging", + debuginfo_level: Option = "debuginfo-level", + debuginfo_level_rustc: Option = "debuginfo-level-rustc", + debuginfo_level_std: Option = "debuginfo-level-std", + debuginfo_level_tools: Option = "debuginfo-level-tools", + debuginfo_level_tests: Option = "debuginfo-level-tests", + split_debuginfo: Option = "split-debuginfo", + run_dsymutil: Option = "run-dsymutil", + backtrace: Option = "backtrace", + incremental: Option = "incremental", + parallel_compiler: Option = "parallel-compiler", + default_linker: Option = "default-linker", + channel: Option = "channel", + description: Option = "description", + musl_root: Option = "musl-root", + rpath: Option = "rpath", + verbose_tests: Option = "verbose-tests", + optimize_tests: Option = "optimize-tests", + codegen_tests: Option = "codegen-tests", + omit_git_hash: Option = "omit-git-hash", + dist_src: Option = "dist-src", + save_toolstates: Option = "save-toolstates", + codegen_backends: Option> = "codegen-backends", + lld: Option = "lld", + use_lld: Option = "use-lld", + llvm_tools: Option = "llvm-tools", + deny_warnings: Option = "deny-warnings", + backtrace_on_ice: Option = "backtrace-on-ice", + verify_llvm_ir: Option = "verify-llvm-ir", + thin_lto_import_instr_limit: Option = "thin-lto-import-instr-limit", + remap_debuginfo: Option = "remap-debuginfo", + jemalloc: Option = "jemalloc", + test_compare_mode: Option = "test-compare-mode", + llvm_libunwind: Option = "llvm-libunwind", + control_flow_guard: Option = "control-flow-guard", + new_symbol_mangling: Option = "new-symbol-mangling", + profile_generate: Option = "profile-generate", + profile_use: Option = "profile-use", + // ignored; this is set from an env var set by bootstrap.py + download_rustc: Option = "download-rustc", + lto: Option = "lto", + validate_mir_opts: Option = "validate-mir-opts", + } +} + +define_config! { + /// TOML representation of how each build target is configured. + struct TomlTarget { + cc: Option = "cc", + cxx: Option = "cxx", + ar: Option = "ar", + ranlib: Option = "ranlib", + default_linker: Option = "default-linker", + linker: Option = "linker", + llvm_config: Option = "llvm-config", + llvm_has_rust_patches: Option = "llvm-has-rust-patches", + llvm_filecheck: Option = "llvm-filecheck", + llvm_libunwind: Option = "llvm-libunwind", + sanitizers: Option = "sanitizers", + profiler: Option = "profiler", + rpath: Option = "rpath", + crt_static: Option = "crt-static", + musl_root: Option = "musl-root", + musl_libdir: Option = "musl-libdir", + wasi_root: Option = "wasi-root", + qemu_rootfs: Option = "qemu-rootfs", + no_std: Option = "no-std", + } +} + +impl Config { + pub fn default_opts() -> Config { + let mut config = Config::default(); + config.llvm_optimize = true; + config.ninja_in_file = true; + config.llvm_static_stdcpp = false; + config.backtrace = true; + config.rust_optimize = RustOptimize::Bool(true); + config.rust_optimize_tests = true; + config.submodules = None; + config.docs = true; + config.docs_minification = true; + config.rust_rpath = true; + config.channel = "dev".to_string(); + config.codegen_tests = true; + config.rust_dist_src = true; + config.rust_codegen_backends = vec![INTERNER.intern_str("llvm")]; + config.deny_warnings = true; + config.bindir = "bin".into(); + config.dist_include_mingw_linker = true; + config.dist_compression_profile = "fast".into(); + config.rustc_parallel = true; + + config.stdout_is_tty = std::io::stdout().is_terminal(); + config.stderr_is_tty = std::io::stderr().is_terminal(); + + // set by build.rs + config.build = TargetSelection::from_user(&env!("BUILD_TRIPLE")); + + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + // Undo `src/bootstrap` + config.src = manifest_dir.parent().unwrap().parent().unwrap().to_owned(); + config.out = PathBuf::from("build"); + + config + } + + pub fn parse(args: &[String]) -> Config { + #[cfg(test)] + fn get_toml(_: &Path) -> TomlConfig { + TomlConfig::default() + } + + #[cfg(not(test))] + fn get_toml(file: &Path) -> TomlConfig { + let contents = + t!(fs::read_to_string(file), format!("config file {} not found", file.display())); + // Deserialize to Value and then TomlConfig to prevent the Deserialize impl of + // TomlConfig and sub types to be monomorphized 5x by toml. + toml::from_str(&contents) + .and_then(|table: toml::Value| TomlConfig::deserialize(table)) + .unwrap_or_else(|err| { + eprintln!("failed to parse TOML configuration '{}': {err}", file.display()); + exit!(2); + }) + } + Self::parse_inner(args, get_toml) + } + + fn parse_inner(args: &[String], get_toml: impl Fn(&Path) -> TomlConfig) -> Config { + let mut flags = Flags::parse(&args); + let mut config = Config::default_opts(); + + // Set flags. + config.paths = std::mem::take(&mut flags.paths); + config.skip = flags.skip.into_iter().chain(flags.exclude).collect(); + config.include_default_paths = flags.include_default_paths; + config.rustc_error_format = flags.rustc_error_format; + config.json_output = flags.json_output; + config.on_fail = flags.on_fail; + config.jobs = Some(threads_from_config(flags.jobs as u32)); + config.cmd = flags.cmd; + config.incremental = flags.incremental; + config.dry_run = if flags.dry_run { DryRun::UserSelected } else { DryRun::Disabled }; + config.keep_stage = flags.keep_stage; + config.keep_stage_std = flags.keep_stage_std; + config.color = flags.color; + config.free_args = std::mem::take(&mut flags.free_args); + config.llvm_profile_use = flags.llvm_profile_use; + config.llvm_profile_generate = flags.llvm_profile_generate; + config.enable_bolt_settings = flags.enable_bolt_settings; + + // Infer the rest of the configuration. + + // Infer the source directory. This is non-trivial because we want to support a downloaded bootstrap binary, + // running on a completely machine from where it was compiled. + let mut cmd = Command::new("git"); + // NOTE: we cannot support running from outside the repository because the only path we have available + // is set at compile time, which can be wrong if bootstrap was downloaded from source. + // We still support running outside the repository if we find we aren't in a git directory. + cmd.arg("rev-parse").arg("--show-toplevel"); + // Discard stderr because we expect this to fail when building from a tarball. + let output = cmd + .stderr(std::process::Stdio::null()) + .output() + .ok() + .and_then(|output| if output.status.success() { Some(output) } else { None }); + if let Some(output) = output { + let git_root = String::from_utf8(output.stdout).unwrap(); + // We need to canonicalize this path to make sure it uses backslashes instead of forward slashes. + let git_root = PathBuf::from(git_root.trim()).canonicalize().unwrap(); + let s = git_root.to_str().unwrap(); + + // Bootstrap is quite bad at handling /? in front of paths + let src = match s.strip_prefix("\\\\?\\") { + Some(p) => PathBuf::from(p), + None => PathBuf::from(git_root), + }; + // If this doesn't have at least `stage0.json`, we guessed wrong. This can happen when, + // for example, the build directory is inside of another unrelated git directory. + // In that case keep the original `CARGO_MANIFEST_DIR` handling. + // + // NOTE: this implies that downloadable bootstrap isn't supported when the build directory is outside + // the source directory. We could fix that by setting a variable from all three of python, ./x, and x.ps1. + if src.join("src").join("stage0.json").exists() { + config.src = src; + } + } else { + // We're building from a tarball, not git sources. + // We don't support pre-downloaded bootstrap in this case. + } + + if cfg!(test) { + // Use the build directory of the original x.py invocation, so that we can set `initial_rustc` properly. + config.out = Path::new( + &env::var_os("CARGO_TARGET_DIR").expect("cargo test directly is not supported"), + ) + .parent() + .unwrap() + .to_path_buf(); + } + + let stage0_json = t!(std::fs::read(&config.src.join("src").join("stage0.json"))); + + config.stage0_metadata = t!(serde_json::from_slice::(&stage0_json)); + + // Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory. + let toml_path = flags + .config + .clone() + .or_else(|| env::var_os("RUST_BOOTSTRAP_CONFIG").map(PathBuf::from)); + let using_default_path = toml_path.is_none(); + let mut toml_path = toml_path.unwrap_or_else(|| PathBuf::from("config.toml")); + if using_default_path && !toml_path.exists() { + toml_path = config.src.join(toml_path); + } + + // Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path, + // but not if `config.toml` hasn't been created. + let mut toml = if !using_default_path || toml_path.exists() { + config.config = Some(toml_path.clone()); + get_toml(&toml_path) + } else { + config.config = None; + TomlConfig::default() + }; + + if let Some(include) = &toml.profile { + // Allows creating alias for profile names, allowing + // profiles to be renamed while maintaining back compatibility + // Keep in sync with `profile_aliases` in bootstrap.py + let profile_aliases = HashMap::from([("user", "dist")]); + let include = match profile_aliases.get(include.as_str()) { + Some(alias) => alias, + None => include.as_str(), + }; + let mut include_path = config.src.clone(); + include_path.push("src"); + include_path.push("bootstrap"); + include_path.push("defaults"); + include_path.push(format!("config.{include}.toml")); + let included_toml = get_toml(&include_path); + toml.merge(included_toml, ReplaceOpt::IgnoreDuplicate); + } + + let mut override_toml = TomlConfig::default(); + for option in flags.set.iter() { + fn get_table(option: &str) -> Result { + toml::from_str(&option) + .and_then(|table: toml::Value| TomlConfig::deserialize(table)) + } + + let mut err = match get_table(option) { + Ok(v) => { + override_toml.merge(v, ReplaceOpt::ErrorOnDuplicate); + continue; + } + Err(e) => e, + }; + // We want to be able to set string values without quotes, + // like in `configure.py`. Try adding quotes around the right hand side + if let Some((key, value)) = option.split_once("=") { + if !value.contains('"') { + match get_table(&format!(r#"{key}="{value}""#)) { + Ok(v) => { + override_toml.merge(v, ReplaceOpt::ErrorOnDuplicate); + continue; + } + Err(e) => err = e, + } + } + } + eprintln!("failed to parse override `{option}`: `{err}"); + exit!(2) + } + toml.merge(override_toml, ReplaceOpt::Override); + + config.changelog_seen = toml.changelog_seen; + config.change_id = toml.change_id; + + let build = toml.build.unwrap_or_default(); + if let Some(file_build) = build.build { + config.build = TargetSelection::from_user(&file_build); + }; + + set(&mut config.out, flags.build_dir.or_else(|| build.build_dir.map(PathBuf::from))); + // NOTE: Bootstrap spawns various commands with different working directories. + // To avoid writing to random places on the file system, `config.out` needs to be an absolute path. + if !config.out.is_absolute() { + // `canonicalize` requires the path to already exist. Use our vendored copy of `absolute` instead. + config.out = crate::utils::helpers::absolute(&config.out); + } + + config.initial_rustc = if let Some(rustc) = build.rustc { + if !flags.skip_stage0_validation { + config.check_build_rustc_version(&rustc); + } + PathBuf::from(rustc) + } else { + config.download_beta_toolchain(); + config.out.join(config.build.triple).join("stage0/bin/rustc") + }; + + config.initial_cargo = build + .cargo + .map(|cargo| { + t!(PathBuf::from(cargo).canonicalize(), "`initial_cargo` not found on disk") + }) + .unwrap_or_else(|| config.out.join(config.build.triple).join("stage0/bin/cargo")); + + // NOTE: it's important this comes *after* we set `initial_rustc` just above. + if config.dry_run() { + let dir = config.out.join("tmp-dry-run"); + t!(fs::create_dir_all(&dir)); + config.out = dir; + } + + config.hosts = if let Some(TargetSelectionList(arg_host)) = flags.host { + arg_host + } else if let Some(file_host) = build.host { + file_host.iter().map(|h| TargetSelection::from_user(h)).collect() + } else { + vec![config.build] + }; + config.targets = if let Some(TargetSelectionList(arg_target)) = flags.target { + arg_target + } else if let Some(file_target) = build.target { + file_target.iter().map(|h| TargetSelection::from_user(h)).collect() + } else { + // If target is *not* configured, then default to the host + // toolchains. + config.hosts.clone() + }; + + config.nodejs = build.nodejs.map(PathBuf::from); + config.npm = build.npm.map(PathBuf::from); + config.gdb = build.gdb.map(PathBuf::from); + config.python = build.python.map(PathBuf::from); + config.reuse = build.reuse.map(PathBuf::from); + config.submodules = build.submodules; + config.android_ndk = build.android_ndk; + set(&mut config.low_priority, build.low_priority); + set(&mut config.compiler_docs, build.compiler_docs); + set(&mut config.library_docs_private_items, build.library_docs_private_items); + set(&mut config.docs_minification, build.docs_minification); + set(&mut config.docs, build.docs); + set(&mut config.locked_deps, build.locked_deps); + set(&mut config.vendor, build.vendor); + set(&mut config.full_bootstrap, build.full_bootstrap); + set(&mut config.extended, build.extended); + config.tools = build.tools; + set(&mut config.verbose, build.verbose); + set(&mut config.sanitizers, build.sanitizers); + set(&mut config.profiler, build.profiler); + set(&mut config.cargo_native_static, build.cargo_native_static); + set(&mut config.configure_args, build.configure_args); + set(&mut config.local_rebuild, build.local_rebuild); + set(&mut config.print_step_timings, build.print_step_timings); + set(&mut config.print_step_rusage, build.print_step_rusage); + config.patch_binaries_for_nix = build.patch_binaries_for_nix; + + config.verbose = cmp::max(config.verbose, flags.verbose as usize); + + if let Some(install) = toml.install { + config.prefix = install.prefix.map(PathBuf::from); + config.sysconfdir = install.sysconfdir.map(PathBuf::from); + config.datadir = install.datadir.map(PathBuf::from); + config.docdir = install.docdir.map(PathBuf::from); + set(&mut config.bindir, install.bindir.map(PathBuf::from)); + config.libdir = install.libdir.map(PathBuf::from); + config.mandir = install.mandir.map(PathBuf::from); + } + + // Store off these values as options because if they're not provided + // we'll infer default values for them later + let mut llvm_assertions = None; + let mut llvm_tests = None; + let mut llvm_plugins = None; + let mut debug = None; + let mut debug_assertions = None; + let mut debug_assertions_std = None; + let mut overflow_checks = None; + let mut overflow_checks_std = None; + let mut debug_logging = None; + let mut debuginfo_level = None; + let mut debuginfo_level_rustc = None; + let mut debuginfo_level_std = None; + let mut debuginfo_level_tools = None; + let mut debuginfo_level_tests = None; + let mut optimize = None; + let mut omit_git_hash = None; + + if let Some(rust) = toml.rust { + set(&mut config.channel, rust.channel); + + config.download_rustc_commit = config.download_ci_rustc_commit(rust.download_rustc); + // This list is incomplete, please help by expanding it! + if config.download_rustc_commit.is_some() { + // We need the channel used by the downloaded compiler to match the one we set for rustdoc; + // otherwise rustdoc-ui tests break. + let ci_channel = t!(fs::read_to_string(config.src.join("src/ci/channel"))); + let ci_channel = ci_channel.trim_end(); + if config.channel != ci_channel + && !(config.channel == "dev" && ci_channel == "nightly") + { + panic!( + "setting rust.channel={} is incompatible with download-rustc", + config.channel + ); + } + } + + debug = rust.debug; + debug_assertions = rust.debug_assertions; + debug_assertions_std = rust.debug_assertions_std; + overflow_checks = rust.overflow_checks; + overflow_checks_std = rust.overflow_checks_std; + debug_logging = rust.debug_logging; + debuginfo_level = rust.debuginfo_level; + debuginfo_level_rustc = rust.debuginfo_level_rustc; + debuginfo_level_std = rust.debuginfo_level_std; + debuginfo_level_tools = rust.debuginfo_level_tools; + debuginfo_level_tests = rust.debuginfo_level_tests; + + config.rust_split_debuginfo = rust + .split_debuginfo + .as_deref() + .map(SplitDebuginfo::from_str) + .map(|v| v.expect("invalid value for rust.split_debuginfo")) + .unwrap_or(SplitDebuginfo::default_for_platform(&config.build.triple)); + optimize = rust.optimize; + omit_git_hash = rust.omit_git_hash; + config.rust_new_symbol_mangling = rust.new_symbol_mangling; + set(&mut config.rust_optimize_tests, rust.optimize_tests); + set(&mut config.codegen_tests, rust.codegen_tests); + set(&mut config.rust_rpath, rust.rpath); + set(&mut config.jemalloc, rust.jemalloc); + set(&mut config.test_compare_mode, rust.test_compare_mode); + set(&mut config.backtrace, rust.backtrace); + config.description = rust.description; + set(&mut config.rust_dist_src, rust.dist_src); + set(&mut config.verbose_tests, rust.verbose_tests); + // in the case "false" is set explicitly, do not overwrite the command line args + if let Some(true) = rust.incremental { + config.incremental = true; + } + set(&mut config.use_lld, rust.use_lld); + set(&mut config.lld_enabled, rust.lld); + set(&mut config.llvm_tools_enabled, rust.llvm_tools); + config.rustc_parallel = rust + .parallel_compiler + .unwrap_or(config.channel == "dev" || config.channel == "nightly"); + config.rustc_default_linker = rust.default_linker; + config.musl_root = rust.musl_root.map(PathBuf::from); + config.save_toolstates = rust.save_toolstates.map(PathBuf::from); + set( + &mut config.deny_warnings, + match flags.warnings { + Warnings::Deny => Some(true), + Warnings::Warn => Some(false), + Warnings::Default => rust.deny_warnings, + }, + ); + set(&mut config.backtrace_on_ice, rust.backtrace_on_ice); + set(&mut config.rust_verify_llvm_ir, rust.verify_llvm_ir); + config.rust_thin_lto_import_instr_limit = rust.thin_lto_import_instr_limit; + set(&mut config.rust_remap_debuginfo, rust.remap_debuginfo); + set(&mut config.control_flow_guard, rust.control_flow_guard); + config.llvm_libunwind_default = rust + .llvm_libunwind + .map(|v| v.parse().expect("failed to parse rust.llvm-libunwind")); + + if let Some(ref backends) = rust.codegen_backends { + let available_backends = vec!["llvm", "cranelift", "gcc"]; + + config.rust_codegen_backends = backends.iter().map(|s| { + if let Some(backend) = s.strip_prefix(CODEGEN_BACKEND_PREFIX) { + if available_backends.contains(&backend) { + panic!("Invalid value '{s}' for 'rust.codegen-backends'. Instead, please use '{backend}'."); + } else { + println!("HELP: '{s}' for 'rust.codegen-backends' might fail. \ + Codegen backends are mostly defined without the '{CODEGEN_BACKEND_PREFIX}' prefix. \ + In this case, it would be referred to as '{backend}'."); + } + } + + INTERNER.intern_str(s) + }).collect(); + } + + config.rust_codegen_units = rust.codegen_units.map(threads_from_config); + config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config); + config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use); + config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate); + config.rust_lto = rust + .lto + .as_deref() + .map(|value| RustcLto::from_str(value).unwrap()) + .unwrap_or_default(); + config.rust_validate_mir_opts = rust.validate_mir_opts; + } else { + config.rust_profile_use = flags.rust_profile_use; + config.rust_profile_generate = flags.rust_profile_generate; + } + + config.reproducible_artifacts = flags.reproducible_artifact; + + // rust_info must be set before is_ci_llvm_available() is called. + let default = config.channel == "dev"; + config.omit_git_hash = omit_git_hash.unwrap_or(default); + config.rust_info = GitInfo::new(config.omit_git_hash, &config.src); + + if let Some(llvm) = toml.llvm { + match llvm.ccache { + Some(StringOrBool::String(ref s)) => config.ccache = Some(s.to_string()), + Some(StringOrBool::Bool(true)) => { + config.ccache = Some("ccache".to_string()); + } + Some(StringOrBool::Bool(false)) | None => {} + } + set(&mut config.ninja_in_file, llvm.ninja); + llvm_assertions = llvm.assertions; + llvm_tests = llvm.tests; + llvm_plugins = llvm.plugins; + set(&mut config.llvm_optimize, llvm.optimize); + set(&mut config.llvm_thin_lto, llvm.thin_lto); + set(&mut config.llvm_release_debuginfo, llvm.release_debuginfo); + set(&mut config.llvm_static_stdcpp, llvm.static_libstdcpp); + if let Some(v) = llvm.link_shared { + config.llvm_link_shared.set(Some(v)); + } + config.llvm_targets = llvm.targets.clone(); + config.llvm_experimental_targets = llvm.experimental_targets.clone(); + config.llvm_link_jobs = llvm.link_jobs; + config.llvm_version_suffix = llvm.version_suffix.clone(); + config.llvm_clang_cl = llvm.clang_cl.clone(); + + config.llvm_cflags = llvm.cflags.clone(); + config.llvm_cxxflags = llvm.cxxflags.clone(); + config.llvm_ldflags = llvm.ldflags.clone(); + set(&mut config.llvm_use_libcxx, llvm.use_libcxx); + config.llvm_use_linker = llvm.use_linker.clone(); + config.llvm_allow_old_toolchain = llvm.allow_old_toolchain.unwrap_or(false); + config.llvm_polly = llvm.polly.unwrap_or(false); + config.llvm_clang = llvm.clang.unwrap_or(false); + config.llvm_enable_warnings = llvm.enable_warnings.unwrap_or(false); + config.llvm_build_config = llvm.build_config.clone().unwrap_or(Default::default()); + + let asserts = llvm_assertions.unwrap_or(false); + config.llvm_from_ci = config.parse_download_ci_llvm(llvm.download_ci_llvm, asserts); + + if config.llvm_from_ci { + // None of the LLVM options, except assertions, are supported + // when using downloaded LLVM. We could just ignore these but + // that's potentially confusing, so force them to not be + // explicitly set. The defaults and CI defaults don't + // necessarily match but forcing people to match (somewhat + // arbitrary) CI configuration locally seems bad/hard. + check_ci_llvm!(llvm.optimize); + check_ci_llvm!(llvm.thin_lto); + check_ci_llvm!(llvm.release_debuginfo); + // CI-built LLVM can be either dynamic or static. We won't know until we download it. + check_ci_llvm!(llvm.link_shared); + check_ci_llvm!(llvm.static_libstdcpp); + check_ci_llvm!(llvm.targets); + check_ci_llvm!(llvm.experimental_targets); + check_ci_llvm!(llvm.link_jobs); + check_ci_llvm!(llvm.clang_cl); + check_ci_llvm!(llvm.version_suffix); + check_ci_llvm!(llvm.cflags); + check_ci_llvm!(llvm.cxxflags); + check_ci_llvm!(llvm.ldflags); + check_ci_llvm!(llvm.use_libcxx); + check_ci_llvm!(llvm.use_linker); + check_ci_llvm!(llvm.allow_old_toolchain); + check_ci_llvm!(llvm.polly); + check_ci_llvm!(llvm.clang); + check_ci_llvm!(llvm.build_config); + check_ci_llvm!(llvm.plugins); + } + + // NOTE: can never be hit when downloading from CI, since we call `check_ci_llvm!(thin_lto)` above. + if config.llvm_thin_lto && llvm.link_shared.is_none() { + // If we're building with ThinLTO on, by default we want to link + // to LLVM shared, to avoid re-doing ThinLTO (which happens in + // the link step) with each stage. + config.llvm_link_shared.set(Some(true)); + } + } else { + config.llvm_from_ci = config.channel == "dev" + && crate::core::build_steps::llvm::is_ci_llvm_available(&config, false); + } + + if let Some(t) = toml.target { + for (triple, cfg) in t { + let mut target = Target::from_triple(&triple); + + if let Some(ref s) = cfg.llvm_config { + if config.download_rustc_commit.is_some() && triple == &*config.build.triple { + panic!( + "setting llvm_config for the host is incompatible with download-rustc" + ); + } + target.llvm_config = Some(config.src.join(s)); + } + target.llvm_has_rust_patches = cfg.llvm_has_rust_patches; + if let Some(ref s) = cfg.llvm_filecheck { + target.llvm_filecheck = Some(config.src.join(s)); + } + target.llvm_libunwind = cfg + .llvm_libunwind + .as_ref() + .map(|v| v.parse().expect("failed to parse rust.llvm-libunwind")); + if let Some(s) = cfg.no_std { + target.no_std = s; + } + target.cc = cfg.cc.map(PathBuf::from); + target.cxx = cfg.cxx.map(PathBuf::from); + target.ar = cfg.ar.map(PathBuf::from); + target.ranlib = cfg.ranlib.map(PathBuf::from); + target.linker = cfg.linker.map(PathBuf::from); + target.crt_static = cfg.crt_static; + target.musl_root = cfg.musl_root.map(PathBuf::from); + target.musl_libdir = cfg.musl_libdir.map(PathBuf::from); + target.wasi_root = cfg.wasi_root.map(PathBuf::from); + target.qemu_rootfs = cfg.qemu_rootfs.map(PathBuf::from); + target.sanitizers = cfg.sanitizers; + target.profiler = cfg.profiler; + target.rpath = cfg.rpath; + + config.target_config.insert(TargetSelection::from_user(&triple), target); + } + } + + if config.llvm_from_ci { + let triple = &config.build.triple; + let ci_llvm_bin = config.ci_llvm_root().join("bin"); + let build_target = config + .target_config + .entry(config.build) + .or_insert_with(|| Target::from_triple(&triple)); + + check_ci_llvm!(build_target.llvm_config); + check_ci_llvm!(build_target.llvm_filecheck); + build_target.llvm_config = Some(ci_llvm_bin.join(exe("llvm-config", config.build))); + build_target.llvm_filecheck = Some(ci_llvm_bin.join(exe("FileCheck", config.build))); + } + + if let Some(t) = toml.dist { + config.dist_sign_folder = t.sign_folder.map(PathBuf::from); + config.dist_upload_addr = t.upload_addr; + config.dist_compression_formats = t.compression_formats; + set(&mut config.dist_compression_profile, t.compression_profile); + set(&mut config.rust_dist_src, t.src_tarball); + set(&mut config.missing_tools, t.missing_tools); + set(&mut config.dist_include_mingw_linker, t.include_mingw_linker) + } + + if let Some(r) = build.rustfmt { + *config.initial_rustfmt.borrow_mut() = if r.exists() { + RustfmtState::SystemToolchain(r) + } else { + RustfmtState::Unavailable + }; + } + + // Now that we've reached the end of our configuration, infer the + // default values for all options that we haven't otherwise stored yet. + + config.llvm_assertions = llvm_assertions.unwrap_or(false); + config.llvm_tests = llvm_tests.unwrap_or(false); + config.llvm_plugins = llvm_plugins.unwrap_or(false); + config.rust_optimize = optimize.unwrap_or(RustOptimize::Bool(true)); + + let default = debug == Some(true); + config.rust_debug_assertions = debug_assertions.unwrap_or(default); + config.rust_debug_assertions_std = + debug_assertions_std.unwrap_or(config.rust_debug_assertions); + config.rust_overflow_checks = overflow_checks.unwrap_or(default); + config.rust_overflow_checks_std = + overflow_checks_std.unwrap_or(config.rust_overflow_checks); + + config.rust_debug_logging = debug_logging.unwrap_or(config.rust_debug_assertions); + + let with_defaults = |debuginfo_level_specific: Option<_>| { + debuginfo_level_specific.or(debuginfo_level).unwrap_or(if debug == Some(true) { + DebuginfoLevel::Limited + } else { + DebuginfoLevel::None + }) + }; + config.rust_debuginfo_level_rustc = with_defaults(debuginfo_level_rustc); + config.rust_debuginfo_level_std = with_defaults(debuginfo_level_std); + config.rust_debuginfo_level_tools = with_defaults(debuginfo_level_tools); + config.rust_debuginfo_level_tests = debuginfo_level_tests.unwrap_or(DebuginfoLevel::None); + + let download_rustc = config.download_rustc_commit.is_some(); + // See https://github.com/rust-lang/compiler-team/issues/326 + config.stage = match config.cmd { + Subcommand::Check { .. } => flags.stage.or(build.check_stage).unwrap_or(0), + // `download-rustc` only has a speed-up for stage2 builds. Default to stage2 unless explicitly overridden. + Subcommand::Doc { .. } => { + flags.stage.or(build.doc_stage).unwrap_or(if download_rustc { 2 } else { 0 }) + } + Subcommand::Build { .. } => { + flags.stage.or(build.build_stage).unwrap_or(if download_rustc { 2 } else { 1 }) + } + Subcommand::Test { .. } => { + flags.stage.or(build.test_stage).unwrap_or(if download_rustc { 2 } else { 1 }) + } + Subcommand::Bench { .. } => flags.stage.or(build.bench_stage).unwrap_or(2), + Subcommand::Dist { .. } => flags.stage.or(build.dist_stage).unwrap_or(2), + Subcommand::Install { .. } => flags.stage.or(build.install_stage).unwrap_or(2), + // These are all bootstrap tools, which don't depend on the compiler. + // The stage we pass shouldn't matter, but use 0 just in case. + Subcommand::Clean { .. } + | Subcommand::Clippy { .. } + | Subcommand::Fix { .. } + | Subcommand::Run { .. } + | Subcommand::Setup { .. } + | Subcommand::Format { .. } + | Subcommand::Suggest { .. } => flags.stage.unwrap_or(0), + }; + + // CI should always run stage 2 builds, unless it specifically states otherwise + #[cfg(not(test))] + if flags.stage.is_none() && crate::CiEnv::current() != crate::CiEnv::None { + match config.cmd { + Subcommand::Test { .. } + | Subcommand::Doc { .. } + | Subcommand::Build { .. } + | Subcommand::Bench { .. } + | Subcommand::Dist { .. } + | Subcommand::Install { .. } => { + assert_eq!( + config.stage, 2, + "x.py should be run with `--stage 2` on CI, but was run with `--stage {}`", + config.stage, + ); + } + Subcommand::Clean { .. } + | Subcommand::Check { .. } + | Subcommand::Clippy { .. } + | Subcommand::Fix { .. } + | Subcommand::Run { .. } + | Subcommand::Setup { .. } + | Subcommand::Format { .. } + | Subcommand::Suggest { .. } => {} + } + } + + config + } + + pub(crate) fn dry_run(&self) -> bool { + match self.dry_run { + DryRun::Disabled => false, + DryRun::SelfCheck | DryRun::UserSelected => true, + } + } + + /// Runs a command, printing out nice contextual information if it fails. + /// Exits if the command failed to execute at all, otherwise returns its + /// `status.success()`. + #[deprecated = "use `Builder::try_run` instead where possible"] + pub(crate) fn try_run(&self, cmd: &mut Command) -> Result<(), ()> { + if self.dry_run() { + return Ok(()); + } + self.verbose(&format!("running: {cmd:?}")); + build_helper::util::try_run(cmd, self.is_verbose()) + } + + /// A git invocation which runs inside the source directory. + /// + /// Use this rather than `Command::new("git")` in order to support out-of-tree builds. + pub(crate) fn git(&self) -> Command { + let mut git = Command::new("git"); + git.current_dir(&self.src); + git + } + + pub(crate) fn test_args(&self) -> Vec<&str> { + let mut test_args = match self.cmd { + Subcommand::Test { ref test_args, .. } | Subcommand::Bench { ref test_args, .. } => { + test_args.iter().flat_map(|s| s.split_whitespace()).collect() + } + _ => vec![], + }; + test_args.extend(self.free_args.iter().map(|s| s.as_str())); + test_args + } + + pub(crate) fn args(&self) -> Vec<&str> { + let mut args = match self.cmd { + Subcommand::Run { ref args, .. } => { + args.iter().flat_map(|s| s.split_whitespace()).collect() + } + _ => vec![], + }; + args.extend(self.free_args.iter().map(|s| s.as_str())); + args + } + + /// Bootstrap embeds a version number into the name of shared libraries it uploads in CI. + /// Return the version it would have used for the given commit. + pub(crate) fn artifact_version_part(&self, commit: &str) -> String { + let (channel, version) = if self.rust_info.is_managed_git_subrepository() { + let mut channel = self.git(); + channel.arg("show").arg(format!("{commit}:src/ci/channel")); + let channel = output(&mut channel); + let mut version = self.git(); + version.arg("show").arg(format!("{commit}:src/version")); + let version = output(&mut version); + (channel.trim().to_owned(), version.trim().to_owned()) + } else { + let channel = fs::read_to_string(self.src.join("src/ci/channel")); + let version = fs::read_to_string(self.src.join("src/version")); + match (channel, version) { + (Ok(channel), Ok(version)) => { + (channel.trim().to_owned(), version.trim().to_owned()) + } + (channel, version) => { + let src = self.src.display(); + eprintln!("ERROR: failed to determine artifact channel and/or version"); + eprintln!( + "HELP: consider using a git checkout or ensure these files are readable" + ); + if let Err(channel) = channel { + eprintln!("reading {src}/src/ci/channel failed: {channel:?}"); + } + if let Err(version) = version { + eprintln!("reading {src}/src/version failed: {version:?}"); + } + panic!(); + } + } + }; + + match channel.as_str() { + "stable" => version, + "beta" => channel, + "nightly" => channel, + other => unreachable!("{:?} is not recognized as a valid channel", other), + } + } + + /// Try to find the relative path of `bindir`, otherwise return it in full. + pub fn bindir_relative(&self) -> &Path { + let bindir = &self.bindir; + if bindir.is_absolute() { + // Try to make it relative to the prefix. + if let Some(prefix) = &self.prefix { + if let Ok(stripped) = bindir.strip_prefix(prefix) { + return stripped; + } + } + } + bindir + } + + /// Try to find the relative path of `libdir`. + pub fn libdir_relative(&self) -> Option<&Path> { + let libdir = self.libdir.as_ref()?; + if libdir.is_relative() { + Some(libdir) + } else { + // Try to make it relative to the prefix. + libdir.strip_prefix(self.prefix.as_ref()?).ok() + } + } + + /// The absolute path to the downloaded LLVM artifacts. + pub(crate) fn ci_llvm_root(&self) -> PathBuf { + assert!(self.llvm_from_ci); + self.out.join(&*self.build.triple).join("ci-llvm") + } + + /// Directory where the extracted `rustc-dev` component is stored. + pub(crate) fn ci_rustc_dir(&self) -> PathBuf { + assert!(self.download_rustc()); + self.out.join(self.build.triple).join("ci-rustc") + } + + /// Determine whether llvm should be linked dynamically. + /// + /// If `false`, llvm should be linked statically. + /// This is computed on demand since LLVM might have to first be downloaded from CI. + pub(crate) fn llvm_link_shared(&self) -> bool { + let mut opt = self.llvm_link_shared.get(); + if opt.is_none() && self.dry_run() { + // just assume static for now - dynamic linking isn't supported on all platforms + return false; + } + + let llvm_link_shared = *opt.get_or_insert_with(|| { + if self.llvm_from_ci { + self.maybe_download_ci_llvm(); + let ci_llvm = self.ci_llvm_root(); + let link_type = t!( + std::fs::read_to_string(ci_llvm.join("link-type.txt")), + format!("CI llvm missing: {}", ci_llvm.display()) + ); + link_type == "dynamic" + } else { + // unclear how thought-through this default is, but it maintains compatibility with + // previous behavior + false + } + }); + self.llvm_link_shared.set(opt); + llvm_link_shared + } + + /// Return whether we will use a downloaded, pre-compiled version of rustc, or just build from source. + pub(crate) fn download_rustc(&self) -> bool { + self.download_rustc_commit().is_some() + } + + pub(crate) fn download_rustc_commit(&self) -> Option<&str> { + static DOWNLOAD_RUSTC: OnceCell> = OnceCell::new(); + if self.dry_run() && DOWNLOAD_RUSTC.get().is_none() { + // avoid trying to actually download the commit + return self.download_rustc_commit.as_deref(); + } + + DOWNLOAD_RUSTC + .get_or_init(|| match &self.download_rustc_commit { + None => None, + Some(commit) => { + self.download_ci_rustc(commit); + Some(commit.clone()) + } + }) + .as_deref() + } + + pub(crate) fn initial_rustfmt(&self) -> Option { + match &mut *self.initial_rustfmt.borrow_mut() { + RustfmtState::SystemToolchain(p) | RustfmtState::Downloaded(p) => Some(p.clone()), + RustfmtState::Unavailable => None, + r @ RustfmtState::LazyEvaluated => { + if self.dry_run() { + return Some(PathBuf::new()); + } + let path = self.maybe_download_rustfmt(); + *r = if let Some(p) = &path { + RustfmtState::Downloaded(p.clone()) + } else { + RustfmtState::Unavailable + }; + path + } + } + } + + pub fn verbose(&self, msg: &str) { + if self.verbose > 0 { + println!("{msg}"); + } + } + + pub fn sanitizers_enabled(&self, target: TargetSelection) -> bool { + self.target_config.get(&target).map(|t| t.sanitizers).flatten().unwrap_or(self.sanitizers) + } + + pub fn any_sanitizers_enabled(&self) -> bool { + self.target_config.values().any(|t| t.sanitizers == Some(true)) || self.sanitizers + } + + pub fn profiler_path(&self, target: TargetSelection) -> Option<&str> { + match self.target_config.get(&target)?.profiler.as_ref()? { + StringOrBool::String(s) => Some(s), + StringOrBool::Bool(_) => None, + } + } + + pub fn profiler_enabled(&self, target: TargetSelection) -> bool { + self.target_config + .get(&target) + .and_then(|t| t.profiler.as_ref()) + .map(StringOrBool::is_string_or_true) + .unwrap_or(self.profiler) + } + + pub fn any_profiler_enabled(&self) -> bool { + self.target_config.values().any(|t| matches!(&t.profiler, Some(p) if p.is_string_or_true())) + || self.profiler + } + + pub fn rpath_enabled(&self, target: TargetSelection) -> bool { + self.target_config.get(&target).map(|t| t.rpath).flatten().unwrap_or(self.rust_rpath) + } + + pub fn llvm_enabled(&self) -> bool { + self.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) + } + + pub fn llvm_libunwind(&self, target: TargetSelection) -> LlvmLibunwind { + self.target_config + .get(&target) + .and_then(|t| t.llvm_libunwind) + .or(self.llvm_libunwind_default) + .unwrap_or(if target.contains("fuchsia") { + LlvmLibunwind::InTree + } else { + LlvmLibunwind::No + }) + } + + pub fn submodules(&self, rust_info: &GitInfo) -> bool { + self.submodules.unwrap_or(rust_info.is_managed_git_subrepository()) + } + + pub fn default_codegen_backend(&self) -> Option> { + self.rust_codegen_backends.get(0).cloned() + } + + pub fn git_config(&self) -> GitConfig<'_> { + GitConfig { + git_repository: &self.stage0_metadata.config.git_repository, + nightly_branch: &self.stage0_metadata.config.nightly_branch, + } + } + + pub fn check_build_rustc_version(&self, rustc_path: &str) { + if self.dry_run() { + return; + } + + // check rustc version is same or lower with 1 apart from the building one + let mut cmd = Command::new(rustc_path); + cmd.arg("--version"); + let rustc_output = output(&mut cmd) + .lines() + .next() + .unwrap() + .split(' ') + .nth(1) + .unwrap() + .split('-') + .next() + .unwrap() + .to_owned(); + let rustc_version = Version::parse(&rustc_output.trim()).unwrap(); + let source_version = + Version::parse(&fs::read_to_string(self.src.join("src/version")).unwrap().trim()) + .unwrap(); + if !(source_version == rustc_version + || (source_version.major == rustc_version.major + && (source_version.minor == rustc_version.minor + || source_version.minor == rustc_version.minor + 1))) + { + let prev_version = format!("{}.{}.x", source_version.major, source_version.minor - 1); + eprintln!( + "Unexpected rustc version: {rustc_version}, we should use {prev_version}/{source_version} to build source with {source_version}" + ); + exit!(1); + } + } + + /// Returns the commit to download, or `None` if we shouldn't download CI artifacts. + fn download_ci_rustc_commit(&self, download_rustc: Option) -> Option { + // If `download-rustc` is not set, default to rebuilding. + let if_unchanged = match download_rustc { + None | Some(StringOrBool::Bool(false)) => return None, + Some(StringOrBool::Bool(true)) => false, + Some(StringOrBool::String(s)) if s == "if-unchanged" => true, + Some(StringOrBool::String(other)) => { + panic!("unrecognized option for download-rustc: {other}") + } + }; + + // Handle running from a directory other than the top level + let top_level = output(self.git().args(&["rev-parse", "--show-toplevel"])); + let top_level = top_level.trim_end(); + let compiler = format!("{top_level}/compiler/"); + let library = format!("{top_level}/library/"); + + // Look for a version to compare to based on the current commit. + // Only commits merged by bors will have CI artifacts. + let merge_base = output( + self.git() + .arg("rev-list") + .arg(format!("--author={}", self.stage0_metadata.config.git_merge_commit_email)) + .args(&["-n1", "--first-parent", "HEAD"]), + ); + let commit = merge_base.trim_end(); + if commit.is_empty() { + println!("ERROR: could not find commit hash for downloading rustc"); + println!("HELP: maybe your repository history is too shallow?"); + println!("HELP: consider disabling `download-rustc`"); + println!("HELP: or fetch enough history to include one upstream commit"); + crate::exit!(1); + } + + // Warn if there were changes to the compiler or standard library since the ancestor commit. + let has_changes = !t!(self + .git() + .args(&["diff-index", "--quiet", &commit, "--", &compiler, &library]) + .status()) + .success(); + if has_changes { + if if_unchanged { + if self.verbose > 0 { + println!( + "WARNING: saw changes to compiler/ or library/ since {commit}; \ + ignoring `download-rustc`" + ); + } + return None; + } + println!( + "WARNING: `download-rustc` is enabled, but there are changes to \ + compiler/ or library/" + ); + } + + Some(commit.to_string()) + } + + fn parse_download_ci_llvm( + &self, + download_ci_llvm: Option, + asserts: bool, + ) -> bool { + match download_ci_llvm { + None => self.channel == "dev" && llvm::is_ci_llvm_available(&self, asserts), + Some(StringOrBool::Bool(b)) => b, + Some(StringOrBool::String(s)) if s == "if-available" => { + llvm::is_ci_llvm_available(&self, asserts) + } + Some(StringOrBool::String(s)) if s == "if-unchanged" => { + // Git is needed to track modifications here, but tarball source is not available. + // If not modified here or built through tarball source, we maintain consistency + // with '"if available"'. + if !self.rust_info.is_from_tarball() + && self + .last_modified_commit(&["src/llvm-project"], "download-ci-llvm", true) + .is_none() + { + // there are some untracked changes in the the given paths. + false + } else { + llvm::is_ci_llvm_available(&self, asserts) + } + } + Some(StringOrBool::String(other)) => { + panic!("unrecognized option for download-ci-llvm: {:?}", other) + } + } + } + + /// Returns the last commit in which any of `modified_paths` were changed, + /// or `None` if there are untracked changes in the working directory and `if_unchanged` is true. + pub fn last_modified_commit( + &self, + modified_paths: &[&str], + option_name: &str, + if_unchanged: bool, + ) -> Option { + // Handle running from a directory other than the top level + let top_level = output(self.git().args(&["rev-parse", "--show-toplevel"])); + let top_level = top_level.trim_end(); + + // Look for a version to compare to based on the current commit. + // Only commits merged by bors will have CI artifacts. + let merge_base = output( + self.git() + .arg("rev-list") + .arg(format!("--author={}", self.stage0_metadata.config.git_merge_commit_email)) + .args(&["-n1", "--first-parent", "HEAD"]), + ); + let commit = merge_base.trim_end(); + if commit.is_empty() { + println!("error: could not find commit hash for downloading components from CI"); + println!("help: maybe your repository history is too shallow?"); + println!("help: consider disabling `{option_name}`"); + println!("help: or fetch enough history to include one upstream commit"); + crate::exit!(1); + } + + // Warn if there were changes to the compiler or standard library since the ancestor commit. + let mut git = self.git(); + git.args(&["diff-index", "--quiet", &commit, "--"]); + + for path in modified_paths { + git.arg(format!("{top_level}/{path}")); + } + + let has_changes = !t!(git.status()).success(); + if has_changes { + if if_unchanged { + if self.verbose > 0 { + println!( + "warning: saw changes to one of {modified_paths:?} since {commit}; \ + ignoring `{option_name}`" + ); + } + return None; + } + println!( + "warning: `{option_name}` is enabled, but there are changes to one of {modified_paths:?}" + ); + } + + Some(commit.to_string()) + } +} + +fn set(field: &mut T, val: Option) { + if let Some(v) = val { + *field = v; + } +} + +fn threads_from_config(v: u32) -> u32 { + match v { + 0 => std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32, + n => n, + } +} diff --git a/src/bootstrap/src/core/config/flags.rs b/src/bootstrap/src/core/config/flags.rs new file mode 100644 index 000000000..2a301007a --- /dev/null +++ b/src/bootstrap/src/core/config/flags.rs @@ -0,0 +1,571 @@ +//! Command-line interface of the rustbuild build system. +//! +//! This module implements the command-line parsing of the build system which +//! has various flags to configure how it's run. + +use std::path::{Path, PathBuf}; + +use clap::{CommandFactory, Parser, ValueEnum}; + +use crate::core::build_steps::setup::Profile; +use crate::core::builder::{Builder, Kind}; +use crate::core::config::{target_selection_list, Config, TargetSelectionList}; +use crate::{Build, DocTests}; + +#[derive(Copy, Clone, Default, Debug, ValueEnum)] +pub enum Color { + Always, + Never, + #[default] + Auto, +} + +/// Whether to deny warnings, emit them as warnings, or use the default behavior +#[derive(Copy, Clone, Default, Debug, ValueEnum)] +pub enum Warnings { + Deny, + Warn, + #[default] + Default, +} + +/// Deserialized version of all flags for this compile. +#[derive(Debug, Parser)] +#[clap( + override_usage = "x.py [options] [...]", + disable_help_subcommand(true), + about = "", + next_line_help(false) +)] +pub struct Flags { + #[command(subcommand)] + pub cmd: Subcommand, + + #[arg(global(true), short, long, action = clap::ArgAction::Count)] + /// use verbose output (-vv for very verbose) + pub verbose: u8, // each extra -v after the first is passed to Cargo + #[arg(global(true), short, long)] + /// use incremental compilation + pub incremental: bool, + #[arg(global(true), long, value_hint = clap::ValueHint::FilePath, value_name = "FILE")] + /// TOML configuration file for build + pub config: Option, + #[arg(global(true), long, value_hint = clap::ValueHint::DirPath, value_name = "DIR")] + /// Build directory, overrides `build.build-dir` in `config.toml` + pub build_dir: Option, + + #[arg(global(true), long, value_hint = clap::ValueHint::Other, value_name = "BUILD")] + /// build target of the stage0 compiler + pub build: Option, + + #[arg(global(true), long, value_hint = clap::ValueHint::Other, value_name = "HOST", value_parser = target_selection_list)] + /// host targets to build + pub host: Option, + + #[arg(global(true), long, value_hint = clap::ValueHint::Other, value_name = "TARGET", value_parser = target_selection_list)] + /// target targets to build + pub target: Option, + + #[arg(global(true), long, value_name = "PATH")] + /// build paths to exclude + pub exclude: Vec, // keeping for client backward compatibility + #[arg(global(true), long, value_name = "PATH")] + /// build paths to skip + pub skip: Vec, + #[arg(global(true), long)] + /// include default paths in addition to the provided ones + pub include_default_paths: bool, + + #[arg(global(true), value_hint = clap::ValueHint::Other, long)] + pub rustc_error_format: Option, + + #[arg(global(true), long, value_hint = clap::ValueHint::CommandString, value_name = "CMD")] + /// command to run on failure + pub on_fail: Option, + #[arg(global(true), long)] + /// dry run; don't build anything + pub dry_run: bool, + #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "N")] + /// stage to build (indicates compiler to use/test, e.g., stage 0 uses the + /// bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.) + pub stage: Option, + + #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "N")] + /// stage(s) to keep without recompiling + /// (pass multiple times to keep e.g., both stages 0 and 1) + pub keep_stage: Vec, + #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "N")] + /// stage(s) of the standard library to keep without recompiling + /// (pass multiple times to keep e.g., both stages 0 and 1) + pub keep_stage_std: Vec, + #[arg(global(true), long, value_hint = clap::ValueHint::DirPath, value_name = "DIR")] + /// path to the root of the rust checkout + pub src: Option, + + #[arg( + global(true), + short, + long, + value_hint = clap::ValueHint::Other, + default_value_t = std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get), + value_name = "JOBS" + )] + /// number of jobs to run in parallel + pub jobs: usize, + // This overrides the deny-warnings configuration option, + // which passes -Dwarnings to the compiler invocations. + #[arg(global(true), long)] + #[clap(value_enum, default_value_t=Warnings::Default, value_name = "deny|warn")] + /// if value is deny, will deny warnings + /// if value is warn, will emit warnings + /// otherwise, use the default configured behaviour + pub warnings: Warnings, + + #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "FORMAT")] + /// rustc error format + pub error_format: Option, + #[arg(global(true), long)] + /// use message-format=json + pub json_output: bool, + + #[arg(global(true), long, value_name = "STYLE")] + #[clap(value_enum, default_value_t = Color::Auto)] + /// whether to use color in cargo and rustc output + pub color: Color, + + /// whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml + #[arg(global(true), long, value_name = "VALUE")] + pub llvm_skip_rebuild: Option, + /// generate PGO profile with rustc build + #[arg(global(true), value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] + pub rust_profile_generate: Option, + /// use PGO profile for rustc build + #[arg(global(true), value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] + pub rust_profile_use: Option, + /// use PGO profile for LLVM build + #[arg(global(true), value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] + pub llvm_profile_use: Option, + // LLVM doesn't support a custom location for generating profile + // information. + // + // llvm_out/build/profiles/ is the location this writes to. + /// generate PGO profile with llvm built for rustc + #[arg(global(true), long)] + pub llvm_profile_generate: bool, + /// Enable BOLT link flags + #[arg(global(true), long)] + pub enable_bolt_settings: bool, + /// Skip stage0 compiler validation + #[arg(global(true), long)] + pub skip_stage0_validation: bool, + /// Additional reproducible artifacts that should be added to the reproducible artifacts archive. + #[arg(global(true), long)] + pub reproducible_artifact: Vec, + #[arg(global(true))] + /// paths for the subcommand + pub paths: Vec, + /// override options in config.toml + #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "section.option=value")] + pub set: Vec, + /// arguments passed to subcommands + #[arg(global(true), last(true), value_name = "ARGS")] + pub free_args: Vec, +} + +impl Flags { + pub fn parse(args: &[String]) -> Self { + let first = String::from("x.py"); + let it = std::iter::once(&first).chain(args.iter()); + // We need to check for ` -h -v`, in which case we list the paths + #[derive(Parser)] + #[clap(disable_help_flag(true))] + struct HelpVerboseOnly { + #[arg(short, long)] + help: bool, + #[arg(global(true), short, long, action = clap::ArgAction::Count)] + pub verbose: u8, + #[arg(value_enum)] + cmd: Kind, + } + if let Ok(HelpVerboseOnly { help: true, verbose: 1.., cmd: subcommand }) = + HelpVerboseOnly::try_parse_from(it.clone()) + { + println!("NOTE: updating submodules before printing available paths"); + let config = Config::parse(&[String::from("build")]); + let build = Build::new(config); + let paths = Builder::get_help(&build, subcommand); + if let Some(s) = paths { + println!("{s}"); + } else { + panic!("No paths available for subcommand `{}`", subcommand.as_str()); + } + crate::exit!(0); + } + + Flags::parse_from(it) + } +} + +#[derive(Debug, Clone, Default, clap::Subcommand)] +pub enum Subcommand { + #[clap(aliases = ["b"], long_about = "\n + Arguments: + This subcommand accepts a number of paths to directories to the crates + and/or artifacts to compile. For example, for a quick build of a usable + compiler: + ./x.py build --stage 1 library/std + This will build a compiler and standard library from the local source code. + Once this is done, build/$ARCH/stage1 contains a usable compiler. + If no arguments are passed then the default artifacts for that stage are + compiled. For example: + ./x.py build --stage 0 + ./x.py build ")] + /// Compile either the compiler or libraries + #[default] + Build, + #[clap(aliases = ["c"], long_about = "\n + Arguments: + This subcommand accepts a number of paths to directories to the crates + and/or artifacts to compile. For example: + ./x.py check library/std + If no arguments are passed then many artifacts are checked.")] + /// Compile either the compiler or libraries, using cargo check + Check { + #[arg(long)] + /// Check all targets + all_targets: bool, + }, + /// Run Clippy (uses rustup/cargo-installed clippy binary) + #[clap(long_about = "\n + Arguments: + This subcommand accepts a number of paths to directories to the crates + and/or artifacts to run clippy against. For example: + ./x.py clippy library/core + ./x.py clippy library/core library/proc_macro")] + Clippy { + #[arg(long)] + fix: bool, + /// clippy lints to allow + #[arg(global(true), short = 'A', action = clap::ArgAction::Append, value_name = "LINT")] + allow: Vec, + /// clippy lints to deny + #[arg(global(true), short = 'D', action = clap::ArgAction::Append, value_name = "LINT")] + deny: Vec, + /// clippy lints to warn on + #[arg(global(true), short = 'W', action = clap::ArgAction::Append, value_name = "LINT")] + warn: Vec, + /// clippy lints to forbid + #[arg(global(true), short = 'F', action = clap::ArgAction::Append, value_name = "LINT")] + forbid: Vec, + }, + /// Run cargo fix + #[clap(long_about = "\n + Arguments: + This subcommand accepts a number of paths to directories to the crates + and/or artifacts to run `cargo fix` against. For example: + ./x.py fix library/core + ./x.py fix library/core library/proc_macro")] + Fix, + #[clap( + name = "fmt", + long_about = "\n + Arguments: + This subcommand optionally accepts a `--check` flag which succeeds if formatting is correct and + fails if it is not. For example: + ./x.py fmt + ./x.py fmt --check" + )] + /// Run rustfmt + Format { + /// check formatting instead of applying + #[arg(long)] + check: bool, + }, + #[clap(aliases = ["d"], long_about = "\n + Arguments: + This subcommand accepts a number of paths to directories of documentation + to build. For example: + ./x.py doc src/doc/book + ./x.py doc src/doc/nomicon + ./x.py doc src/doc/book library/std + ./x.py doc library/std --json + ./x.py doc library/std --open + If no arguments are passed then everything is documented: + ./x.py doc + ./x.py doc --stage 1")] + /// Build documentation + Doc { + #[arg(long)] + /// open the docs in a browser + open: bool, + #[arg(long)] + /// render the documentation in JSON format in addition to the usual HTML format + json: bool, + }, + #[clap(aliases = ["t"], long_about = "\n + Arguments: + This subcommand accepts a number of paths to test directories that + should be compiled and run. For example: + ./x.py test tests/ui + ./x.py test library/std --test-args hash_map + ./x.py test library/std --stage 0 --no-doc + ./x.py test tests/ui --bless + ./x.py test tests/ui --compare-mode next-solver + Note that `test tests/* --stage N` does NOT depend on `build compiler/rustc --stage N`; + just like `build library/std --stage N` it tests the compiler produced by the previous + stage. + Execute tool tests with a tool name argument: + ./x.py test tidy + If no arguments are passed then the complete artifacts for that stage are + compiled and tested. + ./x.py test + ./x.py test --stage 1")] + /// Build and run some test suites + Test { + #[arg(long)] + /// run all tests regardless of failure + no_fail_fast: bool, + #[arg(long, value_name = "SUBSTRING")] + /// skips tests matching SUBSTRING, if supported by test tool. May be passed multiple times + skip: Vec, + #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] + /// extra arguments to be passed for the test tool being used + /// (e.g. libtest, compiletest or rustdoc) + test_args: Vec, + /// extra options to pass the compiler when running tests + #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] + rustc_args: Vec, + #[arg(long)] + /// do not run doc tests + no_doc: bool, + #[arg(long)] + /// only run doc tests + doc: bool, + #[arg(long)] + /// whether to automatically update stderr/stdout files + bless: bool, + #[arg(long)] + /// comma-separated list of other files types to check (accepts py, py:lint, + /// py:fmt, shell) + extra_checks: Option, + #[arg(long)] + /// rerun tests even if the inputs are unchanged + force_rerun: bool, + #[arg(long)] + /// only run tests that result has been changed + only_modified: bool, + #[arg(long, value_name = "COMPARE MODE")] + /// mode describing what file the actual ui output will be compared to + compare_mode: Option, + #[arg(long, value_name = "check | build | run")] + /// force {check,build,run}-pass tests to this mode. + pass: Option, + #[arg(long, value_name = "auto | always | never")] + /// whether to execute run-* tests + run: Option, + #[arg(long)] + /// enable this to generate a Rustfix coverage file, which is saved in + /// `//rustfix_missing_coverage.txt` + rustfix_coverage: bool, + }, + /// Build and run some benchmarks + Bench { + #[arg(long, allow_hyphen_values(true))] + test_args: Vec, + }, + /// Clean out build directories + Clean { + #[arg(long)] + /// Clean the entire build directory (not used by default) + all: bool, + #[arg(long, value_name = "N")] + /// Clean a specific stage without touching other artifacts. By default, every stage is cleaned if this option is not used. + stage: Option, + }, + /// Build distribution artifacts + Dist, + /// Install distribution artifacts + Install, + #[clap(aliases = ["r"], long_about = "\n + Arguments: + This subcommand accepts a number of paths to tools to build and run. For + example: + ./x.py run src/tools/expand-yaml-anchors + At least a tool needs to be called.")] + /// Run tools contained in this repository + Run { + /// arguments for the tool + #[arg(long, allow_hyphen_values(true))] + args: Vec, + }, + /// Set up the environment for development + #[clap(long_about = format!( + "\n +x.py setup creates a `config.toml` which changes the defaults for x.py itself, +as well as setting up a git pre-push hook, VS Code config and toolchain link. +Arguments: + This subcommand accepts a 'profile' to use for builds. For example: + ./x.py setup library + The profile is optional and you will be prompted interactively if it is not given. + The following profiles are available: +{} + To only set up the git hook, VS Code config or toolchain link, you may use + ./x.py setup hook + ./x.py setup vscode + ./x.py setup link", Profile::all_for_help(" ").trim_end()))] + Setup { + /// Either the profile for `config.toml` or another setup action. + /// May be omitted to set up interactively + #[arg(value_name = "|hook|vscode|link")] + profile: Option, + }, + /// Suggest a subset of tests to run, based on modified files + #[clap(long_about = "\n")] + Suggest { + /// run suggested tests + #[arg(long)] + run: bool, + }, +} + +impl Subcommand { + pub fn kind(&self) -> Kind { + match self { + Subcommand::Bench { .. } => Kind::Bench, + Subcommand::Build { .. } => Kind::Build, + Subcommand::Check { .. } => Kind::Check, + Subcommand::Clippy { .. } => Kind::Clippy, + Subcommand::Doc { .. } => Kind::Doc, + Subcommand::Fix { .. } => Kind::Fix, + Subcommand::Format { .. } => Kind::Format, + Subcommand::Test { .. } => Kind::Test, + Subcommand::Clean { .. } => Kind::Clean, + Subcommand::Dist { .. } => Kind::Dist, + Subcommand::Install { .. } => Kind::Install, + Subcommand::Run { .. } => Kind::Run, + Subcommand::Setup { .. } => Kind::Setup, + Subcommand::Suggest { .. } => Kind::Suggest, + } + } + + pub fn rustc_args(&self) -> Vec<&str> { + match *self { + Subcommand::Test { ref rustc_args, .. } => { + rustc_args.iter().flat_map(|s| s.split_whitespace()).collect() + } + _ => vec![], + } + } + + pub fn fail_fast(&self) -> bool { + match *self { + Subcommand::Test { no_fail_fast, .. } => !no_fail_fast, + _ => false, + } + } + + pub fn doc_tests(&self) -> DocTests { + match *self { + Subcommand::Test { doc, no_doc, .. } => { + if doc { + DocTests::Only + } else if no_doc { + DocTests::No + } else { + DocTests::Yes + } + } + _ => DocTests::Yes, + } + } + + pub fn bless(&self) -> bool { + match *self { + Subcommand::Test { bless, .. } => bless, + _ => false, + } + } + + pub fn extra_checks(&self) -> Option<&str> { + match *self { + Subcommand::Test { ref extra_checks, .. } => extra_checks.as_ref().map(String::as_str), + _ => None, + } + } + + pub fn only_modified(&self) -> bool { + match *self { + Subcommand::Test { only_modified, .. } => only_modified, + _ => false, + } + } + + pub fn force_rerun(&self) -> bool { + match *self { + Subcommand::Test { force_rerun, .. } => force_rerun, + _ => false, + } + } + + pub fn rustfix_coverage(&self) -> bool { + match *self { + Subcommand::Test { rustfix_coverage, .. } => rustfix_coverage, + _ => false, + } + } + + pub fn compare_mode(&self) -> Option<&str> { + match *self { + Subcommand::Test { ref compare_mode, .. } => compare_mode.as_ref().map(|s| &s[..]), + _ => None, + } + } + + pub fn pass(&self) -> Option<&str> { + match *self { + Subcommand::Test { ref pass, .. } => pass.as_ref().map(|s| &s[..]), + _ => None, + } + } + + pub fn run(&self) -> Option<&str> { + match *self { + Subcommand::Test { ref run, .. } => run.as_ref().map(|s| &s[..]), + _ => None, + } + } + + pub fn open(&self) -> bool { + match *self { + Subcommand::Doc { open, .. } => open, + _ => false, + } + } + + pub fn json(&self) -> bool { + match *self { + Subcommand::Doc { json, .. } => json, + _ => false, + } + } +} + +/// Returns the shell completion for a given shell, if the result differs from the current +/// content of `path`. If `path` does not exist, always returns `Some`. +pub fn get_completion(shell: G, path: &Path) -> Option { + let mut cmd = Flags::command(); + let current = if !path.exists() { + String::new() + } else { + std::fs::read_to_string(path).unwrap_or_else(|_| { + eprintln!("couldn't read {}", path.display()); + crate::exit!(1) + }) + }; + let mut buf = Vec::new(); + clap_complete::generate(shell, &mut cmd, "x.py", &mut buf); + if buf == current.as_bytes() { + return None; + } + Some(String::from_utf8(buf).expect("completion script should be UTF-8")) +} diff --git a/src/bootstrap/src/core/config/mod.rs b/src/bootstrap/src/core/config/mod.rs new file mode 100644 index 000000000..9c6861826 --- /dev/null +++ b/src/bootstrap/src/core/config/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod config; +pub(crate) mod flags; + +pub use config::*; diff --git a/src/bootstrap/src/core/download.rs b/src/bootstrap/src/core/download.rs new file mode 100644 index 000000000..3327aed96 --- /dev/null +++ b/src/bootstrap/src/core/download.rs @@ -0,0 +1,701 @@ +use std::{ + env, + ffi::{OsStr, OsString}, + fs::{self, File}, + io::{BufRead, BufReader, BufWriter, ErrorKind, Write}, + path::{Path, PathBuf}, + process::{Command, Stdio}, +}; + +use build_helper::ci::CiEnv; +use once_cell::sync::OnceCell; +use xz2::bufread::XzDecoder; + +use crate::core::build_steps::llvm::detect_llvm_sha; +use crate::core::config::RustfmtMetadata; +use crate::utils::helpers::{check_run, exe, program_out_of_date}; +use crate::{t, Config}; + +static SHOULD_FIX_BINS_AND_DYLIBS: OnceCell = OnceCell::new(); + +/// `Config::try_run` wrapper for this module to avoid warnings on `try_run`, since we don't have access to a `builder` yet. +fn try_run(config: &Config, cmd: &mut Command) -> Result<(), ()> { + #[allow(deprecated)] + config.try_run(cmd) +} + +/// Generic helpers that are useful anywhere in bootstrap. +impl Config { + pub fn is_verbose(&self) -> bool { + self.verbose > 0 + } + + pub(crate) fn create(&self, path: &Path, s: &str) { + if self.dry_run() { + return; + } + t!(fs::write(path, s)); + } + + pub(crate) fn remove(&self, f: &Path) { + if self.dry_run() { + return; + } + fs::remove_file(f).unwrap_or_else(|_| panic!("failed to remove {:?}", f)); + } + + /// Create a temporary directory in `out` and return its path. + /// + /// NOTE: this temporary directory is shared between all steps; + /// if you need an empty directory, create a new subdirectory inside it. + pub(crate) fn tempdir(&self) -> PathBuf { + let tmp = self.out.join("tmp"); + t!(fs::create_dir_all(&tmp)); + tmp + } + + /// Runs a command, printing out nice contextual information if it fails. + /// Returns false if do not execute at all, otherwise returns its + /// `status.success()`. + pub(crate) fn check_run(&self, cmd: &mut Command) -> bool { + if self.dry_run() { + return true; + } + self.verbose(&format!("running: {cmd:?}")); + check_run(cmd, self.is_verbose()) + } + + /// Whether or not `fix_bin_or_dylib` needs to be run; can only be true + /// on NixOS + fn should_fix_bins_and_dylibs(&self) -> bool { + let val = *SHOULD_FIX_BINS_AND_DYLIBS.get_or_init(|| { + match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() { + Err(_) => return false, + Ok(output) if !output.status.success() => return false, + Ok(output) => { + let mut os_name = output.stdout; + if os_name.last() == Some(&b'\n') { + os_name.pop(); + } + if os_name != b"Linux" { + return false; + } + } + } + + // If the user has asked binaries to be patched for Nix, then + // don't check for NixOS or `/lib`. + // NOTE: this intentionally comes after the Linux check: + // - patchelf only works with ELF files, so no need to run it on Mac or Windows + // - On other Unix systems, there is no stable syscall interface, so Nix doesn't manage the global libc. + if let Some(explicit_value) = self.patch_binaries_for_nix { + return explicit_value; + } + + // Use `/etc/os-release` instead of `/etc/NIXOS`. + // The latter one does not exist on NixOS when using tmpfs as root. + let is_nixos = match File::open("/etc/os-release") { + Err(e) if e.kind() == ErrorKind::NotFound => false, + Err(e) => panic!("failed to access /etc/os-release: {}", e), + Ok(os_release) => BufReader::new(os_release).lines().any(|l| { + let l = l.expect("reading /etc/os-release"); + matches!(l.trim(), "ID=nixos" | "ID='nixos'" | "ID=\"nixos\"") + }), + }; + if !is_nixos { + let in_nix_shell = env::var("IN_NIX_SHELL"); + if let Ok(in_nix_shell) = in_nix_shell { + eprintln!( + "The IN_NIX_SHELL environment variable is `{in_nix_shell}`; \ + you may need to set `patch-binaries-for-nix=true` in config.toml" + ); + } + } + is_nixos + }); + if val { + eprintln!("INFO: You seem to be using Nix."); + } + val + } + + /// Modifies the interpreter section of 'fname' to fix the dynamic linker, + /// or the RPATH section, to fix the dynamic library search path + /// + /// This is only required on NixOS and uses the PatchELF utility to + /// change the interpreter/RPATH of ELF executables. + /// + /// Please see for more information + fn fix_bin_or_dylib(&self, fname: &Path) { + assert_eq!(SHOULD_FIX_BINS_AND_DYLIBS.get(), Some(&true)); + println!("attempting to patch {}", fname.display()); + + // Only build `.nix-deps` once. + static NIX_DEPS_DIR: OnceCell = OnceCell::new(); + let mut nix_build_succeeded = true; + let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| { + // Run `nix-build` to "build" each dependency (which will likely reuse + // the existing `/nix/store` copy, or at most download a pre-built copy). + // + // Importantly, we create a gc-root called `.nix-deps` in the `build/` + // directory, but still reference the actual `/nix/store` path in the rpath + // as it makes it significantly more robust against changes to the location of + // the `.nix-deps` location. + // + // bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`). + // zlib: Needed as a system dependency of `libLLVM-*.so`. + // patchelf: Needed for patching ELF binaries (see doc comment above). + let nix_deps_dir = self.out.join(".nix-deps"); + const NIX_EXPR: &str = " + with (import {}); + symlinkJoin { + name = \"rust-stage0-dependencies\"; + paths = [ + zlib + patchelf + stdenv.cc.bintools + ]; + } + "; + nix_build_succeeded = try_run( + self, + Command::new("nix-build").args(&[ + Path::new("-E"), + Path::new(NIX_EXPR), + Path::new("-o"), + &nix_deps_dir, + ]), + ) + .is_ok(); + nix_deps_dir + }); + if !nix_build_succeeded { + return; + } + + let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf")); + let rpath_entries = { + // ORIGIN is a relative default, all binary and dynamic libraries we ship + // appear to have this (even when `../lib` is redundant). + // NOTE: there are only two paths here, delimited by a `:` + let mut entries = OsString::from("$ORIGIN/../lib:"); + entries.push(t!(fs::canonicalize(nix_deps_dir)).join("lib")); + entries + }; + patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]); + if !fname.extension().map_or(false, |ext| ext == "so") { + // Finally, set the correct .interp for binaries + let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker"); + // FIXME: can we support utf8 here? `args` doesn't accept Vec, only OsString ... + let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path)))); + patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]); + } + + let _ = try_run(self, patchelf.arg(fname)); + } + + fn download_file(&self, url: &str, dest_path: &Path, help_on_error: &str) { + self.verbose(&format!("download {url}")); + // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/. + let tempfile = self.tempdir().join(dest_path.file_name().unwrap()); + // While bootstrap itself only supports http and https downloads, downstream forks might + // need to download components from other protocols. The match allows them adding more + // protocols without worrying about merge conflicts if we change the HTTP implementation. + match url.split_once("://").map(|(proto, _)| proto) { + Some("http") | Some("https") => { + self.download_http_with_retries(&tempfile, url, help_on_error) + } + Some(other) => panic!("unsupported protocol {other} in {url}"), + None => panic!("no protocol in {url}"), + } + t!(std::fs::rename(&tempfile, dest_path)); + } + + fn download_http_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) { + println!("downloading {url}"); + // Try curl. If that fails and we are on windows, fallback to PowerShell. + let mut curl = Command::new("curl"); + curl.args(&[ + "-y", + "30", + "-Y", + "10", // timeout if speed is < 10 bytes/sec for > 30 seconds + "--connect-timeout", + "30", // timeout if cannot connect within 30 seconds + "-o", + tempfile.to_str().unwrap(), + "--retry", + "3", + "-SRf", + ]); + // Don't print progress in CI; the \r wrapping looks bad and downloads don't take long enough for progress to be useful. + if CiEnv::is_ci() { + curl.arg("-s"); + } else { + curl.arg("--progress-bar"); + } + curl.arg(url); + if !self.check_run(&mut curl) { + if self.build.contains("windows-msvc") { + eprintln!("Fallback to PowerShell"); + for _ in 0..3 { + if try_run(self, Command::new("PowerShell.exe").args(&[ + "/nologo", + "-Command", + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", + &format!( + "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", + url, tempfile.to_str().expect("invalid UTF-8 not supported with powershell downloads"), + ), + ])).is_err() { + return; + } + eprintln!("\nspurious failure, trying again"); + } + } + if !help_on_error.is_empty() { + eprintln!("{help_on_error}"); + } + crate::exit!(1); + } + } + + fn unpack(&self, tarball: &Path, dst: &Path, pattern: &str) { + eprintln!("extracting {} to {}", tarball.display(), dst.display()); + if !dst.exists() { + t!(fs::create_dir_all(dst)); + } + + // `tarball` ends with `.tar.xz`; strip that suffix + // example: `rust-dev-nightly-x86_64-unknown-linux-gnu` + let uncompressed_filename = + Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap(); + let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap()); + + // decompress the file + let data = t!(File::open(tarball), format!("file {} not found", tarball.display())); + let decompressor = XzDecoder::new(BufReader::new(data)); + + let mut tar = tar::Archive::new(decompressor); + + // `compile::Sysroot` needs to know the contents of the `rustc-dev` tarball to avoid adding + // it to the sysroot unless it was explicitly requested. But parsing the 100 MB tarball is slow. + // Cache the entries when we extract it so we only have to read it once. + let mut recorded_entries = + if dst.ends_with("ci-rustc") { recorded_entries(dst, pattern) } else { None }; + + for member in t!(tar.entries()) { + let mut member = t!(member); + let original_path = t!(member.path()).into_owned(); + // skip the top-level directory + if original_path == directory_prefix { + continue; + } + let mut short_path = t!(original_path.strip_prefix(directory_prefix)); + if !short_path.starts_with(pattern) { + continue; + } + short_path = t!(short_path.strip_prefix(pattern)); + let dst_path = dst.join(short_path); + self.verbose(&format!("extracting {} to {}", original_path.display(), dst.display())); + if !t!(member.unpack_in(dst)) { + panic!("path traversal attack ??"); + } + if let Some(record) = &mut recorded_entries { + t!(writeln!(record, "{}", short_path.to_str().unwrap())); + } + let src_path = dst.join(original_path); + if src_path.is_dir() && dst_path.exists() { + continue; + } + t!(fs::rename(src_path, dst_path)); + } + let dst_dir = dst.join(directory_prefix); + if dst_dir.exists() { + t!(fs::remove_dir_all(&dst_dir), format!("failed to remove {}", dst_dir.display())); + } + } + + /// Returns whether the SHA256 checksum of `path` matches `expected`. + pub(crate) fn verify(&self, path: &Path, expected: &str) -> bool { + use sha2::Digest; + + self.verbose(&format!("verifying {}", path.display())); + + if self.dry_run() { + return false; + } + + let mut hasher = sha2::Sha256::new(); + + let file = t!(File::open(path)); + let mut reader = BufReader::new(file); + + loop { + let buffer = t!(reader.fill_buf()); + let l = buffer.len(); + // break if EOF + if l == 0 { + break; + } + hasher.update(buffer); + reader.consume(l); + } + + let checksum = hex::encode(hasher.finalize().as_slice()); + let verified = checksum == expected; + + if !verified { + println!( + "invalid checksum: \n\ + found: {checksum}\n\ + expected: {expected}", + ); + } + + verified + } +} + +fn recorded_entries(dst: &Path, pattern: &str) -> Option> { + let name = if pattern == "rustc-dev" { + ".rustc-dev-contents" + } else if pattern.starts_with("rust-std") { + ".rust-std-contents" + } else { + return None; + }; + Some(BufWriter::new(t!(File::create(dst.join(name))))) +} + +enum DownloadSource { + CI, + Dist, +} + +/// Functions that are only ever called once, but named for clarify and to avoid thousand-line functions. +impl Config { + pub(crate) fn maybe_download_rustfmt(&self) -> Option { + let RustfmtMetadata { date, version } = self.stage0_metadata.rustfmt.as_ref()?; + let channel = format!("{version}-{date}"); + + let host = self.build; + let bin_root = self.out.join(host.triple).join("rustfmt"); + let rustfmt_path = bin_root.join("bin").join(exe("rustfmt", host)); + let rustfmt_stamp = bin_root.join(".rustfmt-stamp"); + if rustfmt_path.exists() && !program_out_of_date(&rustfmt_stamp, &channel) { + return Some(rustfmt_path); + } + + self.download_component( + DownloadSource::Dist, + format!("rustfmt-{version}-{build}.tar.xz", build = host.triple), + "rustfmt-preview", + &date, + "rustfmt", + ); + self.download_component( + DownloadSource::Dist, + format!("rustc-{version}-{build}.tar.xz", build = host.triple), + "rustc", + &date, + "rustfmt", + ); + + if self.should_fix_bins_and_dylibs() { + self.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt")); + self.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt")); + let lib_dir = bin_root.join("lib"); + for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) { + let lib = t!(lib); + if lib.path().extension() == Some(OsStr::new("so")) { + self.fix_bin_or_dylib(&lib.path()); + } + } + } + + self.create(&rustfmt_stamp, &channel); + Some(rustfmt_path) + } + + pub(crate) fn ci_rust_std_contents(&self) -> Vec { + self.ci_component_contents(".rust-std-contents") + } + + pub(crate) fn ci_rustc_dev_contents(&self) -> Vec { + self.ci_component_contents(".rustc-dev-contents") + } + + fn ci_component_contents(&self, stamp_file: &str) -> Vec { + assert!(self.download_rustc()); + if self.dry_run() { + return vec![]; + } + + let ci_rustc_dir = self.ci_rustc_dir(); + let stamp_file = ci_rustc_dir.join(stamp_file); + let contents_file = t!(File::open(&stamp_file), stamp_file.display().to_string()); + t!(BufReader::new(contents_file).lines().collect()) + } + + pub(crate) fn download_ci_rustc(&self, commit: &str) { + self.verbose(&format!("using downloaded stage2 artifacts from CI (commit {commit})")); + + let version = self.artifact_version_part(commit); + // download-rustc doesn't need its own cargo, it can just use beta's. But it does need the + // `rustc_private` crates for tools. + let extra_components = ["rustc-dev"]; + + self.download_toolchain( + &version, + "ci-rustc", + &format!("{commit}-{}", self.llvm_assertions), + &extra_components, + Self::download_ci_component, + ); + } + + pub(crate) fn download_beta_toolchain(&self) { + self.verbose("downloading stage0 beta artifacts"); + + let date = &self.stage0_metadata.compiler.date; + let version = &self.stage0_metadata.compiler.version; + let extra_components = ["cargo"]; + + let download_beta_component = |config: &Config, filename, prefix: &_, date: &_| { + config.download_component(DownloadSource::Dist, filename, prefix, date, "stage0") + }; + + self.download_toolchain( + version, + "stage0", + date, + &extra_components, + download_beta_component, + ); + } + + fn download_toolchain( + &self, + version: &str, + sysroot: &str, + stamp_key: &str, + extra_components: &[&str], + download_component: fn(&Config, String, &str, &str), + ) { + let host = self.build.triple; + let bin_root = self.out.join(host).join(sysroot); + let rustc_stamp = bin_root.join(".rustc-stamp"); + + if !bin_root.join("bin").join(exe("rustc", self.build)).exists() + || program_out_of_date(&rustc_stamp, stamp_key) + { + if bin_root.exists() { + t!(fs::remove_dir_all(&bin_root)); + } + let filename = format!("rust-std-{version}-{host}.tar.xz"); + let pattern = format!("rust-std-{host}"); + download_component(self, filename, &pattern, stamp_key); + let filename = format!("rustc-{version}-{host}.tar.xz"); + download_component(self, filename, "rustc", stamp_key); + + for component in extra_components { + let filename = format!("{component}-{version}-{host}.tar.xz"); + download_component(self, filename, component, stamp_key); + } + + if self.should_fix_bins_and_dylibs() { + self.fix_bin_or_dylib(&bin_root.join("bin").join("rustc")); + self.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc")); + self.fix_bin_or_dylib( + &bin_root.join("libexec").join("rust-analyzer-proc-macro-srv"), + ); + let lib_dir = bin_root.join("lib"); + for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) { + let lib = t!(lib); + if lib.path().extension() == Some(OsStr::new("so")) { + self.fix_bin_or_dylib(&lib.path()); + } + } + } + + t!(fs::write(rustc_stamp, stamp_key)); + } + } + + /// Download a single component of a CI-built toolchain (not necessarily a published nightly). + // NOTE: intentionally takes an owned string to avoid downloading multiple times by accident + fn download_ci_component(&self, filename: String, prefix: &str, commit_with_assertions: &str) { + Self::download_component( + self, + DownloadSource::CI, + filename, + prefix, + commit_with_assertions, + "ci-rustc", + ) + } + + fn download_component( + &self, + mode: DownloadSource, + filename: String, + prefix: &str, + key: &str, + destination: &str, + ) { + let cache_dst = self.out.join("cache"); + let cache_dir = cache_dst.join(key); + if !cache_dir.exists() { + t!(fs::create_dir_all(&cache_dir)); + } + + let bin_root = self.out.join(self.build.triple).join(destination); + let tarball = cache_dir.join(&filename); + let (base_url, url, should_verify) = match mode { + DownloadSource::CI => { + let dist_server = if self.llvm_assertions { + self.stage0_metadata.config.artifacts_with_llvm_assertions_server.clone() + } else { + self.stage0_metadata.config.artifacts_server.clone() + }; + let url = format!( + "{}/{filename}", + key.strip_suffix(&format!("-{}", self.llvm_assertions)).unwrap() + ); + (dist_server, url, false) + } + DownloadSource::Dist => { + let dist_server = env::var("RUSTUP_DIST_SERVER") + .unwrap_or(self.stage0_metadata.config.dist_server.to_string()); + // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json + (dist_server, format!("dist/{key}/{filename}"), true) + } + }; + + // For the beta compiler, put special effort into ensuring the checksums are valid. + // FIXME: maybe we should do this for download-rustc as well? but it would be a pain to update + // this on each and every nightly ... + let checksum = if should_verify { + let error = format!( + "src/stage0.json doesn't contain a checksum for {url}. \ + Pre-built artifacts might not be available for this \ + target at this time, see https://doc.rust-lang.org/nightly\ + /rustc/platform-support.html for more information." + ); + let sha256 = self.stage0_metadata.checksums_sha256.get(&url).expect(&error); + if tarball.exists() { + if self.verify(&tarball, sha256) { + self.unpack(&tarball, &bin_root, prefix); + return; + } else { + self.verbose(&format!( + "ignoring cached file {} due to failed verification", + tarball.display() + )); + self.remove(&tarball); + } + } + Some(sha256) + } else if tarball.exists() { + self.unpack(&tarball, &bin_root, prefix); + return; + } else { + None + }; + + let mut help_on_error = ""; + if destination == "ci-rustc" { + help_on_error = "ERROR: failed to download pre-built rustc from CI + +NOTE: old builds get deleted after a certain time +HELP: if trying to compile an old commit of rustc, disable `download-rustc` in config.toml: + +[rust] +download-rustc = false +"; + } + self.download_file(&format!("{base_url}/{url}"), &tarball, help_on_error); + if let Some(sha256) = checksum { + if !self.verify(&tarball, sha256) { + panic!("failed to verify {}", tarball.display()); + } + } + + self.unpack(&tarball, &bin_root, prefix); + } + + pub(crate) fn maybe_download_ci_llvm(&self) { + if !self.llvm_from_ci { + return; + } + let llvm_root = self.ci_llvm_root(); + let llvm_stamp = llvm_root.join(".llvm-stamp"); + let llvm_sha = detect_llvm_sha(&self, self.rust_info.is_managed_git_subrepository()); + let key = format!("{}{}", llvm_sha, self.llvm_assertions); + if program_out_of_date(&llvm_stamp, &key) && !self.dry_run() { + self.download_ci_llvm(&llvm_sha); + if self.should_fix_bins_and_dylibs() { + for entry in t!(fs::read_dir(llvm_root.join("bin"))) { + self.fix_bin_or_dylib(&t!(entry).path()); + } + } + + // Update the timestamp of llvm-config to force rustc_llvm to be + // rebuilt. This is a hacky workaround for a deficiency in Cargo where + // the rerun-if-changed directive doesn't handle changes very well. + // https://github.com/rust-lang/cargo/issues/10791 + // Cargo only compares the timestamp of the file relative to the last + // time `rustc_llvm` build script ran. However, the timestamps of the + // files in the tarball are in the past, so it doesn't trigger a + // rebuild. + let now = filetime::FileTime::from_system_time(std::time::SystemTime::now()); + let llvm_config = llvm_root.join("bin").join(exe("llvm-config", self.build)); + t!(filetime::set_file_times(&llvm_config, now, now)); + + if self.should_fix_bins_and_dylibs() { + let llvm_lib = llvm_root.join("lib"); + for entry in t!(fs::read_dir(&llvm_lib)) { + let lib = t!(entry).path(); + if lib.extension().map_or(false, |ext| ext == "so") { + self.fix_bin_or_dylib(&lib); + } + } + } + + t!(fs::write(llvm_stamp, key)); + } + } + + fn download_ci_llvm(&self, llvm_sha: &str) { + let llvm_assertions = self.llvm_assertions; + + let cache_prefix = format!("llvm-{llvm_sha}-{llvm_assertions}"); + let cache_dst = self.out.join("cache"); + let rustc_cache = cache_dst.join(cache_prefix); + if !rustc_cache.exists() { + t!(fs::create_dir_all(&rustc_cache)); + } + let base = if llvm_assertions { + &self.stage0_metadata.config.artifacts_with_llvm_assertions_server + } else { + &self.stage0_metadata.config.artifacts_server + }; + let version = self.artifact_version_part(llvm_sha); + let filename = format!("rust-dev-{}-{}.tar.xz", version, self.build.triple); + let tarball = rustc_cache.join(&filename); + if !tarball.exists() { + let help_on_error = "ERROR: failed to download llvm from ci + + HELP: old builds get deleted after a certain time + HELP: if trying to compile an old commit of rustc, disable `download-ci-llvm` in config.toml: + + [llvm] + download-ci-llvm = false + "; + self.download_file(&format!("{base}/{llvm_sha}/{filename}"), &tarball, help_on_error); + } + let llvm_root = self.ci_llvm_root(); + self.unpack(&tarball, &llvm_root, "rust-dev"); + } +} diff --git a/src/bootstrap/src/core/metadata.rs b/src/bootstrap/src/core/metadata.rs new file mode 100644 index 000000000..580208232 --- /dev/null +++ b/src/bootstrap/src/core/metadata.rs @@ -0,0 +1,101 @@ +use std::path::PathBuf; +use std::process::Command; + +use serde_derive::Deserialize; + +use crate::utils::cache::INTERNER; +use crate::utils::helpers::output; +use crate::{t, Build, Crate}; + +/// For more information, see the output of +/// +#[derive(Debug, Deserialize)] +struct Output { + packages: Vec, +} + +/// For more information, see the output of +/// +#[derive(Debug, Deserialize)] +struct Package { + name: String, + source: Option, + manifest_path: String, + dependencies: Vec, + targets: Vec, +} + +/// For more information, see the output of +/// +#[derive(Debug, Deserialize)] +struct Dependency { + name: String, + source: Option, +} + +#[derive(Debug, Deserialize)] +struct Target { + kind: Vec, +} + +/// Collects and stores package metadata of each workspace members into `build`, +/// by executing `cargo metadata` commands. +pub fn build(build: &mut Build) { + for package in workspace_members(build) { + if package.source.is_none() { + let name = INTERNER.intern_string(package.name); + let mut path = PathBuf::from(package.manifest_path); + path.pop(); + let deps = package + .dependencies + .into_iter() + .filter(|dep| dep.source.is_none()) + .map(|dep| INTERNER.intern_string(dep.name)) + .collect(); + let has_lib = package.targets.iter().any(|t| t.kind.iter().any(|k| k == "lib")); + let krate = Crate { name, deps, path, has_lib }; + let relative_path = krate.local_path(build); + build.crates.insert(name, krate); + let existing_path = build.crate_paths.insert(relative_path, name); + assert!( + existing_path.is_none(), + "multiple crates with the same path: {}", + existing_path.unwrap() + ); + } + } +} + +/// Invokes `cargo metadata` to get package metadata of each workspace member. +/// +/// Note that `src/tools/cargo` is no longer a workspace member but we still +/// treat it as one here, by invoking an additional `cargo metadata` command. +fn workspace_members(build: &Build) -> impl Iterator { + let collect_metadata = |manifest_path| { + let mut cargo = Command::new(&build.initial_cargo); + cargo + // Will read the libstd Cargo.toml + // which uses the unstable `public-dependency` feature. + .env("RUSTC_BOOTSTRAP", "1") + .arg("metadata") + .arg("--format-version") + .arg("1") + .arg("--no-deps") + .arg("--manifest-path") + .arg(build.src.join(manifest_path)); + let metadata_output = output(&mut cargo); + let Output { packages, .. } = t!(serde_json::from_str(&metadata_output)); + packages + }; + + // Collects `metadata.packages` from all workspaces. + let packages = collect_metadata("Cargo.toml"); + let cargo_packages = collect_metadata("src/tools/cargo/Cargo.toml"); + let ra_packages = collect_metadata("src/tools/rust-analyzer/Cargo.toml"); + let bootstrap_packages = collect_metadata("src/bootstrap/Cargo.toml"); + + // We only care about the root package from `src/tool/cargo` workspace. + let cargo_package = cargo_packages.into_iter().find(|pkg| pkg.name == "cargo").into_iter(); + + packages.into_iter().chain(cargo_package).chain(ra_packages).chain(bootstrap_packages) +} diff --git a/src/bootstrap/src/core/mod.rs b/src/bootstrap/src/core/mod.rs new file mode 100644 index 000000000..9e18d6704 --- /dev/null +++ b/src/bootstrap/src/core/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod build_steps; +pub(crate) mod builder; +pub(crate) mod config; +pub(crate) mod download; +pub(crate) mod metadata; +pub(crate) mod sanity; diff --git a/src/bootstrap/src/core/sanity.rs b/src/bootstrap/src/core/sanity.rs new file mode 100644 index 000000000..eec3be66a --- /dev/null +++ b/src/bootstrap/src/core/sanity.rs @@ -0,0 +1,267 @@ +//! Sanity checking performed by rustbuild before actually executing anything. +//! +//! This module contains the implementation of ensuring that the build +//! environment looks reasonable before progressing. This will verify that +//! various programs like git and python exist, along with ensuring that all C +//! compilers for cross-compiling are found. +//! +//! In theory if we get past this phase it's a bug if a build fails, but in +//! practice that's likely not true! + +use std::collections::HashMap; +use std::env; +use std::ffi::{OsStr, OsString}; +use std::fs; +use std::path::PathBuf; +use std::process::Command; + +use crate::core::config::Target; +use crate::utils::cache::INTERNER; +use crate::utils::helpers::output; +use crate::Build; + +pub struct Finder { + cache: HashMap>, + path: OsString, +} + +impl Finder { + pub fn new() -> Self { + Self { cache: HashMap::new(), path: env::var_os("PATH").unwrap_or_default() } + } + + pub fn maybe_have>(&mut self, cmd: S) -> Option { + let cmd: OsString = cmd.into(); + let path = &self.path; + self.cache + .entry(cmd.clone()) + .or_insert_with(|| { + for path in env::split_paths(path) { + let target = path.join(&cmd); + let mut cmd_exe = cmd.clone(); + cmd_exe.push(".exe"); + + if target.is_file() // some/path/git + || path.join(&cmd_exe).exists() // some/path/git.exe + || target.join(&cmd_exe).exists() + // some/path/git/git.exe + { + return Some(target); + } + } + None + }) + .clone() + } + + pub fn must_have>(&mut self, cmd: S) -> PathBuf { + self.maybe_have(&cmd).unwrap_or_else(|| { + panic!("\n\ncouldn't find required command: {:?}\n\n", cmd.as_ref()); + }) + } +} + +pub fn check(build: &mut Build) { + let skip_target_sanity = + env::var_os("BOOTSTRAP_SKIP_TARGET_SANITY").is_some_and(|s| s == "1" || s == "true"); + + let path = env::var_os("PATH").unwrap_or_default(); + // On Windows, quotes are invalid characters for filename paths, and if + // one is present as part of the PATH then that can lead to the system + // being unable to identify the files properly. See + // https://github.com/rust-lang/rust/issues/34959 for more details. + if cfg!(windows) && path.to_string_lossy().contains('\"') { + panic!("PATH contains invalid character '\"'"); + } + + let mut cmd_finder = Finder::new(); + // If we've got a git directory we're gonna need git to update + // submodules and learn about various other aspects. + if build.rust_info().is_managed_git_subrepository() { + cmd_finder.must_have("git"); + } + + // We need cmake, but only if we're actually building LLVM or sanitizers. + let building_llvm = build.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) + && build + .hosts + .iter() + .map(|host| { + build + .config + .target_config + .get(host) + .map(|config| config.llvm_config.is_none()) + .unwrap_or(true) + }) + .any(|build_llvm_ourselves| build_llvm_ourselves); + + let need_cmake = building_llvm || build.config.any_sanitizers_enabled(); + if need_cmake && cmd_finder.maybe_have("cmake").is_none() { + eprintln!( + " +Couldn't find required command: cmake + +You should install cmake, or set `download-ci-llvm = true` in the +`[llvm]` section of `config.toml` to download LLVM rather +than building it. +" + ); + crate::exit!(1); + } + + build.config.python = build + .config + .python + .take() + .map(|p| cmd_finder.must_have(p)) + .or_else(|| env::var_os("BOOTSTRAP_PYTHON").map(PathBuf::from)) // set by bootstrap.py + .or_else(|| cmd_finder.maybe_have("python")) + .or_else(|| cmd_finder.maybe_have("python3")) + .or_else(|| cmd_finder.maybe_have("python2")); + + build.config.nodejs = build + .config + .nodejs + .take() + .map(|p| cmd_finder.must_have(p)) + .or_else(|| cmd_finder.maybe_have("node")) + .or_else(|| cmd_finder.maybe_have("nodejs")); + + build.config.npm = build + .config + .npm + .take() + .map(|p| cmd_finder.must_have(p)) + .or_else(|| cmd_finder.maybe_have("npm")); + + build.config.gdb = build + .config + .gdb + .take() + .map(|p| cmd_finder.must_have(p)) + .or_else(|| cmd_finder.maybe_have("gdb")); + + build.config.reuse = build + .config + .reuse + .take() + .map(|p| cmd_finder.must_have(p)) + .or_else(|| cmd_finder.maybe_have("reuse")); + + // We're gonna build some custom C code here and there, host triples + // also build some C++ shims for LLVM so we need a C++ compiler. + for target in &build.targets { + // On emscripten we don't actually need the C compiler to just + // build the target artifacts, only for testing. For the sake + // of easier bot configuration, just skip detection. + if target.contains("emscripten") { + continue; + } + + // We don't use a C compiler on wasm32 + if target.contains("wasm32") { + continue; + } + + // Some environments don't want or need these tools, such as when testing Miri. + // FIXME: it would be better to refactor this code to split necessary setup from pure sanity + // checks, and have a regular flag for skipping the latter. Also see + // . + if skip_target_sanity { + continue; + } + + if !build.config.dry_run() { + cmd_finder.must_have(build.cc(*target)); + if let Some(ar) = build.ar(*target) { + cmd_finder.must_have(ar); + } + } + } + + for host in &build.hosts { + if !build.config.dry_run() { + cmd_finder.must_have(build.cxx(*host).unwrap()); + } + } + + if build.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) { + // Externally configured LLVM requires FileCheck to exist + let filecheck = build.llvm_filecheck(build.build); + if !filecheck.starts_with(&build.out) && !filecheck.exists() && build.config.codegen_tests { + panic!("FileCheck executable {filecheck:?} does not exist"); + } + } + + for target in &build.targets { + build + .config + .target_config + .entry(*target) + .or_insert_with(|| Target::from_triple(&target.triple)); + + if (target.contains("-none-") || target.contains("nvptx")) + && build.no_std(*target) == Some(false) + { + panic!("All the *-none-* and nvptx* targets are no-std targets") + } + + // Some environments don't want or need these tools, such as when testing Miri. + // FIXME: it would be better to refactor this code to split necessary setup from pure sanity + // checks, and have a regular flag for skipping the latter. Also see + // . + if skip_target_sanity { + continue; + } + + // Make sure musl-root is valid. + if target.contains("musl") && !target.contains("unikraft") { + // If this is a native target (host is also musl) and no musl-root is given, + // fall back to the system toolchain in /usr before giving up + if build.musl_root(*target).is_none() && build.config.build == *target { + let target = build.config.target_config.entry(*target).or_default(); + target.musl_root = Some("/usr".into()); + } + match build.musl_libdir(*target) { + Some(libdir) => { + if fs::metadata(libdir.join("libc.a")).is_err() { + panic!("couldn't find libc.a in musl libdir: {}", libdir.display()); + } + } + None => panic!( + "when targeting MUSL either the rust.musl-root \ + option or the target.$TARGET.musl-root option must \ + be specified in config.toml" + ), + } + } + + if need_cmake && target.contains("msvc") { + // There are three builds of cmake on windows: MSVC, MinGW, and + // Cygwin. The Cygwin build does not have generators for Visual + // Studio, so detect that here and error. + let out = output(Command::new("cmake").arg("--help")); + if !out.contains("Visual Studio") { + panic!( + " +cmake does not support Visual Studio generators. + +This is likely due to it being an msys/cygwin build of cmake, +rather than the required windows version, built using MinGW +or Visual Studio. + +If you are building under msys2 try installing the mingw-w64-x86_64-cmake +package instead of cmake: + +$ pacman -R cmake && pacman -S mingw-w64-x86_64-cmake +" + ); + } + } + } + + if let Some(ref s) = build.config.ccache { + cmd_finder.must_have(s); + } +} diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs new file mode 100644 index 000000000..33b8f1a7c --- /dev/null +++ b/src/bootstrap/src/lib.rs @@ -0,0 +1,1874 @@ +//! Implementation of rustbuild, the Rust build system. +//! +//! This module, and its descendants, are the implementation of the Rust build +//! system. Most of this build system is backed by Cargo but the outer layer +//! here serves as the ability to orchestrate calling Cargo, sequencing Cargo +//! builds, building artifacts like LLVM, etc. The goals of rustbuild are: +//! +//! * To be an easily understandable, easily extensible, and maintainable build +//! system. +//! * Leverage standard tools in the Rust ecosystem to build the compiler, aka +//! crates.io and Cargo. +//! * A standard interface to build across all platforms, including MSVC +//! +//! ## Further information +//! +//! More documentation can be found in each respective module below, and you can +//! also check out the `src/bootstrap/README.md` file for more information. + +use std::cell::{Cell, RefCell}; +use std::collections::{HashMap, HashSet}; +use std::env; +use std::fmt::Display; +use std::fs::{self, File}; +use std::io; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; +use std::str; + +use build_helper::ci::{gha, CiEnv}; +use build_helper::exit; +use build_helper::util::fail; +use filetime::FileTime; +use once_cell::sync::OnceCell; +use termcolor::{ColorChoice, StandardStream, WriteColor}; +use utils::channel::GitInfo; + +use crate::core::builder; +use crate::core::builder::Kind; +use crate::core::config::flags; +use crate::core::config::{DryRun, Target}; +use crate::core::config::{LlvmLibunwind, TargetSelection}; +use crate::utils::cache::{Interned, INTERNER}; +use crate::utils::exec::{BehaviorOnFailure, BootstrapCommand, OutputMode}; +use crate::utils::helpers::{self, dir_is_empty, exe, libdir, mtime, output, symlink_dir}; + +mod core; +mod utils; + +pub use crate::core::builder::PathSet; +pub use crate::core::config::flags::Subcommand; +pub use crate::core::config::Config; + +const LLVM_TOOLS: &[&str] = &[ + "llvm-cov", // used to generate coverage report + "llvm-nm", // used to inspect binaries; it shows symbol names, their sizes and visibility + "llvm-objcopy", // used to transform ELFs into binary format which flashing tools consume + "llvm-objdump", // used to disassemble programs + "llvm-profdata", // used to inspect and merge files generated by profiles + "llvm-readobj", // used to get information from ELFs/objects that the other tools don't provide + "llvm-size", // used to prints the size of the linker sections of a program + "llvm-strip", // used to discard symbols from binary files to reduce their size + "llvm-ar", // used for creating and modifying archive files + "llvm-as", // used to convert LLVM assembly to LLVM bitcode + "llvm-dis", // used to disassemble LLVM bitcode + "llc", // used to compile LLVM bytecode + "opt", // used to optimize LLVM bytecode +]; + +/// LLD file names for all flavors. +const LLD_FILE_NAMES: &[&str] = &["ld.lld", "ld64.lld", "lld-link", "wasm-ld"]; + +/// Keeps track of major changes made to the bootstrap configuration. +/// +/// These values also represent the IDs of the PRs that caused major changes. +/// You can visit `https://github.com/rust-lang/rust/pull/{any-id-from-the-list}` to +/// check for more details regarding each change. +/// +/// If you make any major changes (such as adding new values or changing default values), +/// please ensure that the associated PR ID is added to the end of this list. +/// This is necessary because the list must be sorted by the merge date. +pub const CONFIG_CHANGE_HISTORY: &[usize] = &[115898, 116998, 117435, 116881]; + +/// Extra --check-cfg to add when building +/// (Mode restriction, config name, config values (if any)) +const EXTRA_CHECK_CFGS: &[(Option, &str, Option<&[&'static str]>)] = &[ + (None, "bootstrap", None), + (Some(Mode::Rustc), "parallel_compiler", None), + (Some(Mode::ToolRustc), "parallel_compiler", None), + (Some(Mode::Codegen), "parallel_compiler", None), + (Some(Mode::Std), "stdarch_intel_sde", None), + (Some(Mode::Std), "no_fp_fmt_parse", None), + (Some(Mode::Std), "no_global_oom_handling", None), + (Some(Mode::Std), "no_rc", None), + (Some(Mode::Std), "no_sync", None), + (Some(Mode::Std), "freebsd12", None), + (Some(Mode::Std), "freebsd13", None), + (Some(Mode::Std), "backtrace_in_libstd", None), + /* Extra values not defined in the built-in targets yet, but used in std */ + (Some(Mode::Std), "target_env", Some(&["libnx"])), + // (Some(Mode::Std), "target_os", Some(&[])), + (Some(Mode::Std), "target_arch", Some(&["asmjs", "spirv", "nvptx", "xtensa"])), + /* Extra names used by dependencies */ + // FIXME: Used by serde_json, but we should not be triggering on external dependencies. + (Some(Mode::Rustc), "no_btreemap_remove_entry", None), + (Some(Mode::ToolRustc), "no_btreemap_remove_entry", None), + // FIXME: Used by crossbeam-utils, but we should not be triggering on external dependencies. + (Some(Mode::Rustc), "crossbeam_loom", None), + (Some(Mode::ToolRustc), "crossbeam_loom", None), + // FIXME: Used by proc-macro2, but we should not be triggering on external dependencies. + (Some(Mode::Rustc), "span_locations", None), + (Some(Mode::ToolRustc), "span_locations", None), + // FIXME: Used by rustix, but we should not be triggering on external dependencies. + (Some(Mode::Rustc), "rustix_use_libc", None), + (Some(Mode::ToolRustc), "rustix_use_libc", None), + // FIXME: Used by filetime, but we should not be triggering on external dependencies. + (Some(Mode::Rustc), "emulate_second_only_system", None), + (Some(Mode::ToolRustc), "emulate_second_only_system", None), + // Needed to avoid the need to copy windows.lib into the sysroot. + (Some(Mode::Rustc), "windows_raw_dylib", None), + (Some(Mode::ToolRustc), "windows_raw_dylib", None), +]; + +/// A structure representing a Rust compiler. +/// +/// Each compiler has a `stage` that it is associated with and a `host` that +/// corresponds to the platform the compiler runs on. This structure is used as +/// a parameter to many methods below. +#[derive(Eq, PartialOrd, Ord, PartialEq, Clone, Copy, Hash, Debug)] +pub struct Compiler { + stage: u32, + host: TargetSelection, +} + +#[derive(PartialEq, Eq, Copy, Clone, Debug)] +pub enum DocTests { + /// Run normal tests and doc tests (default). + Yes, + /// Do not run any doc tests. + No, + /// Only run doc tests. + Only, +} + +pub enum GitRepo { + Rustc, + Llvm, +} + +/// Global configuration for the build system. +/// +/// This structure transitively contains all configuration for the build system. +/// All filesystem-encoded configuration is in `config`, all flags are in +/// `flags`, and then parsed or probed information is listed in the keys below. +/// +/// This structure is a parameter of almost all methods in the build system, +/// although most functions are implemented as free functions rather than +/// methods specifically on this structure itself (to make it easier to +/// organize). +#[derive(Clone)] +pub struct Build { + /// User-specified configuration from `config.toml`. + config: Config, + + // Version information + version: String, + + // Properties derived from the above configuration + src: PathBuf, + out: PathBuf, + bootstrap_out: PathBuf, + cargo_info: GitInfo, + rust_analyzer_info: GitInfo, + clippy_info: GitInfo, + miri_info: GitInfo, + rustfmt_info: GitInfo, + in_tree_llvm_info: GitInfo, + local_rebuild: bool, + fail_fast: bool, + doc_tests: DocTests, + verbosity: usize, + + // Targets for which to build + build: TargetSelection, + hosts: Vec, + targets: Vec, + + initial_rustc: PathBuf, + initial_cargo: PathBuf, + initial_lld: PathBuf, + initial_libdir: PathBuf, + initial_sysroot: PathBuf, + + // Runtime state filled in later on + // C/C++ compilers and archiver for all targets + cc: RefCell>, + cxx: RefCell>, + ar: RefCell>, + ranlib: RefCell>, + // Miscellaneous + // allow bidirectional lookups: both name -> path and path -> name + crates: HashMap, Crate>, + crate_paths: HashMap>, + is_sudo: bool, + ci_env: CiEnv, + delayed_failures: RefCell>, + prerelease_version: Cell>, + + #[cfg(feature = "build-metrics")] + metrics: crate::utils::metrics::BuildMetrics, +} + +#[derive(Debug, Clone)] +struct Crate { + name: Interned, + deps: HashSet>, + path: PathBuf, + has_lib: bool, +} + +impl Crate { + fn local_path(&self, build: &Build) -> PathBuf { + self.path.strip_prefix(&build.config.src).unwrap().into() + } +} + +/// When building Rust various objects are handled differently. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum DependencyType { + /// Libraries originating from proc-macros. + Host, + /// Typical Rust libraries. + Target, + /// Non Rust libraries and objects shipped to ease usage of certain targets. + TargetSelfContained, +} + +/// The various "modes" of invoking Cargo. +/// +/// These entries currently correspond to the various output directories of the +/// build system, with each mod generating output in a different directory. +#[derive(Debug, Hash, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Mode { + /// Build the standard library, placing output in the "stageN-std" directory. + Std, + + /// Build librustc, and compiler libraries, placing output in the "stageN-rustc" directory. + Rustc, + + /// Build a codegen backend for rustc, placing the output in the "stageN-codegen" directory. + Codegen, + + /// Build a tool, placing output in the "stage0-bootstrap-tools" + /// directory. This is for miscellaneous sets of tools that are built + /// using the bootstrap stage0 compiler in its entirety (target libraries + /// and all). Typically these tools compile with stable Rust. + ToolBootstrap, + + /// Build a tool which uses the locally built std, placing output in the + /// "stageN-tools" directory. Its usage is quite rare, mainly used by + /// compiletest which needs libtest. + ToolStd, + + /// Build a tool which uses the locally built rustc and the target std, + /// placing the output in the "stageN-tools" directory. This is used for + /// anything that needs a fully functional rustc, such as rustdoc, clippy, + /// cargo, rls, rustfmt, miri, etc. + ToolRustc, +} + +impl Mode { + pub fn is_tool(&self) -> bool { + matches!(self, Mode::ToolBootstrap | Mode::ToolRustc | Mode::ToolStd) + } + + pub fn must_support_dlopen(&self) -> bool { + matches!(self, Mode::Std | Mode::Codegen) + } +} + +pub enum CLang { + C, + Cxx, +} + +macro_rules! forward { + ( $( $fn:ident( $($param:ident: $ty:ty),* ) $( -> $ret:ty)? ),+ $(,)? ) => { + impl Build { + $( fn $fn(&self, $($param: $ty),* ) $( -> $ret)? { + self.config.$fn( $($param),* ) + } )+ + } + } +} + +forward! { + verbose(msg: &str), + is_verbose() -> bool, + create(path: &Path, s: &str), + remove(f: &Path), + tempdir() -> PathBuf, + llvm_link_shared() -> bool, + download_rustc() -> bool, + initial_rustfmt() -> Option, +} + +impl Build { + /// Creates a new set of build configuration from the `flags` on the command + /// line and the filesystem `config`. + /// + /// By default all build output will be placed in the current directory. + pub fn new(mut config: Config) -> Build { + let src = config.src.clone(); + let out = config.out.clone(); + + #[cfg(unix)] + // keep this consistent with the equivalent check in x.py: + // https://github.com/rust-lang/rust/blob/a8a33cf27166d3eabaffc58ed3799e054af3b0c6/src/bootstrap/bootstrap.py#L796-L797 + let is_sudo = match env::var_os("SUDO_USER") { + Some(_sudo_user) => { + // SAFETY: getuid() system call is always successful and no return value is reserved + // to indicate an error. + // + // For more context, see https://man7.org/linux/man-pages/man2/geteuid.2.html + let uid = unsafe { libc::getuid() }; + uid == 0 + } + None => false, + }; + #[cfg(not(unix))] + let is_sudo = false; + + let omit_git_hash = config.omit_git_hash; + let rust_info = GitInfo::new(omit_git_hash, &src); + let cargo_info = GitInfo::new(omit_git_hash, &src.join("src/tools/cargo")); + let rust_analyzer_info = GitInfo::new(omit_git_hash, &src.join("src/tools/rust-analyzer")); + let clippy_info = GitInfo::new(omit_git_hash, &src.join("src/tools/clippy")); + let miri_info = GitInfo::new(omit_git_hash, &src.join("src/tools/miri")); + let rustfmt_info = GitInfo::new(omit_git_hash, &src.join("src/tools/rustfmt")); + + // we always try to use git for LLVM builds + let in_tree_llvm_info = GitInfo::new(false, &src.join("src/llvm-project")); + + let initial_target_libdir_str = if config.dry_run() { + "/dummy/lib/path/to/lib/".to_string() + } else { + output( + Command::new(&config.initial_rustc) + .arg("--target") + .arg(config.build.rustc_target_arg()) + .arg("--print") + .arg("target-libdir"), + ) + }; + let initial_target_dir = Path::new(&initial_target_libdir_str).parent().unwrap(); + let initial_lld = initial_target_dir.join("bin").join("rust-lld"); + + let initial_sysroot = if config.dry_run() { + "/dummy".to_string() + } else { + output(Command::new(&config.initial_rustc).arg("--print").arg("sysroot")) + } + .trim() + .to_string(); + + let initial_libdir = initial_target_dir + .parent() + .unwrap() + .parent() + .unwrap() + .strip_prefix(&initial_sysroot) + .unwrap() + .to_path_buf(); + + let version = std::fs::read_to_string(src.join("src").join("version")) + .expect("failed to read src/version"); + let version = version.trim(); + + let bootstrap_out = std::env::current_exe() + .expect("could not determine path to running process") + .parent() + .unwrap() + .to_path_buf(); + if !bootstrap_out.join(exe("rustc", config.build)).exists() && !cfg!(test) { + // this restriction can be lifted whenever https://github.com/rust-lang/rfcs/pull/3028 is implemented + panic!( + "`rustc` not found in {}, run `cargo build --bins` before `cargo run`", + bootstrap_out.display() + ) + } + + if rust_info.is_from_tarball() && config.description.is_none() { + config.description = Some("built from a source tarball".to_owned()); + } + + let mut build = Build { + initial_rustc: config.initial_rustc.clone(), + initial_cargo: config.initial_cargo.clone(), + initial_lld, + initial_libdir, + initial_sysroot: initial_sysroot.into(), + local_rebuild: config.local_rebuild, + fail_fast: config.cmd.fail_fast(), + doc_tests: config.cmd.doc_tests(), + verbosity: config.verbose, + + build: config.build, + hosts: config.hosts.clone(), + targets: config.targets.clone(), + + config, + version: version.to_string(), + src, + out, + bootstrap_out, + + cargo_info, + rust_analyzer_info, + clippy_info, + miri_info, + rustfmt_info, + in_tree_llvm_info, + cc: RefCell::new(HashMap::new()), + cxx: RefCell::new(HashMap::new()), + ar: RefCell::new(HashMap::new()), + ranlib: RefCell::new(HashMap::new()), + crates: HashMap::new(), + crate_paths: HashMap::new(), + is_sudo, + ci_env: CiEnv::current(), + delayed_failures: RefCell::new(Vec::new()), + prerelease_version: Cell::new(None), + + #[cfg(feature = "build-metrics")] + metrics: crate::utils::metrics::BuildMetrics::init(), + }; + + // If local-rust is the same major.minor as the current version, then force a + // local-rebuild + let local_version_verbose = + output(Command::new(&build.initial_rustc).arg("--version").arg("--verbose")); + let local_release = local_version_verbose + .lines() + .filter_map(|x| x.strip_prefix("release:")) + .next() + .unwrap() + .trim(); + if local_release.split('.').take(2).eq(version.split('.').take(2)) { + build.verbose(&format!("auto-detected local-rebuild {local_release}")); + build.local_rebuild = true; + } + + build.verbose("finding compilers"); + utils::cc_detect::find(&build); + // When running `setup`, the profile is about to change, so any requirements we have now may + // be different on the next invocation. Don't check for them until the next time x.py is + // run. This is ok because `setup` never runs any build commands, so it won't fail if commands are missing. + // + // Similarly, for `setup` we don't actually need submodules or cargo metadata. + if !matches!(build.config.cmd, Subcommand::Setup { .. }) { + build.verbose("running sanity check"); + crate::core::sanity::check(&mut build); + + // Make sure we update these before gathering metadata so we don't get an error about missing + // Cargo.toml files. + let rust_submodules = ["src/tools/cargo", "library/backtrace", "library/stdarch"]; + for s in rust_submodules { + build.update_submodule(Path::new(s)); + } + // Now, update all existing submodules. + build.update_existing_submodules(); + + build.verbose("learning about cargo"); + crate::core::metadata::build(&mut build); + } + + // Make a symbolic link so we can use a consistent directory in the documentation. + let build_triple = build.out.join(&build.build.triple); + t!(fs::create_dir_all(&build_triple)); + let host = build.out.join("host"); + if host.is_symlink() { + // Left over from a previous build; overwrite it. + // This matters if `build.build` has changed between invocations. + #[cfg(windows)] + t!(fs::remove_dir(&host)); + #[cfg(not(windows))] + t!(fs::remove_file(&host)); + } + t!( + symlink_dir(&build.config, &build_triple, &host), + format!("symlink_dir({} => {}) failed", host.display(), build_triple.display()) + ); + + build + } + + // modified from `check_submodule` and `update_submodule` in bootstrap.py + /// Given a path to the directory of a submodule, update it. + /// + /// `relative_path` should be relative to the root of the git repository, not an absolute path. + pub(crate) fn update_submodule(&self, relative_path: &Path) { + if !self.config.submodules(&self.rust_info()) { + return; + } + + let absolute_path = self.config.src.join(relative_path); + + // NOTE: The check for the empty directory is here because when running x.py the first time, + // the submodule won't be checked out. Check it out now so we can build it. + if !GitInfo::new(false, &absolute_path).is_managed_git_subrepository() + && !dir_is_empty(&absolute_path) + { + return; + } + + // check_submodule + let checked_out_hash = + output(Command::new("git").args(&["rev-parse", "HEAD"]).current_dir(&absolute_path)); + // update_submodules + let recorded = output( + Command::new("git") + .args(&["ls-tree", "HEAD"]) + .arg(relative_path) + .current_dir(&self.config.src), + ); + let actual_hash = recorded + .split_whitespace() + .nth(2) + .unwrap_or_else(|| panic!("unexpected output `{}`", recorded)); + + // update_submodule + if actual_hash == checked_out_hash.trim_end() { + // already checked out + return; + } + + println!("Updating submodule {}", relative_path.display()); + self.run( + Command::new("git") + .args(&["submodule", "-q", "sync"]) + .arg(relative_path) + .current_dir(&self.config.src), + ); + + // Try passing `--progress` to start, then run git again without if that fails. + let update = |progress: bool| { + // Git is buggy and will try to fetch submodules from the tracking branch for *this* repository, + // even though that has no relation to the upstream for the submodule. + let current_branch = { + let output = self + .config + .git() + .args(["symbolic-ref", "--short", "HEAD"]) + .stderr(Stdio::inherit()) + .output(); + let output = t!(output); + if output.status.success() { + Some(String::from_utf8(output.stdout).unwrap().trim().to_owned()) + } else { + None + } + }; + + let mut git = self.config.git(); + if let Some(branch) = current_branch { + // If there is a tag named after the current branch, git will try to disambiguate by prepending `heads/` to the branch name. + // This syntax isn't accepted by `branch.{branch}`. Strip it. + let branch = branch.strip_prefix("heads/").unwrap_or(&branch); + git.arg("-c").arg(format!("branch.{branch}.remote=origin")); + } + git.args(&["submodule", "update", "--init", "--recursive", "--depth=1"]); + if progress { + git.arg("--progress"); + } + git.arg(relative_path); + git + }; + // NOTE: doesn't use `try_run` because this shouldn't print an error if it fails. + if !update(true).status().map_or(false, |status| status.success()) { + self.run(&mut update(false)); + } + + // Save any local changes, but avoid running `git stash pop` if there are none (since it will exit with an error). + // diff-index reports the modifications through the exit status + let has_local_modifications = !self.run_cmd( + BootstrapCommand::from( + Command::new("git") + .args(&["diff-index", "--quiet", "HEAD"]) + .current_dir(&absolute_path), + ) + .allow_failure() + .output_mode(match self.is_verbose() { + true => OutputMode::PrintAll, + false => OutputMode::PrintOutput, + }), + ); + if has_local_modifications { + self.run(Command::new("git").args(&["stash", "push"]).current_dir(&absolute_path)); + } + + self.run(Command::new("git").args(&["reset", "-q", "--hard"]).current_dir(&absolute_path)); + self.run(Command::new("git").args(&["clean", "-qdfx"]).current_dir(&absolute_path)); + + if has_local_modifications { + self.run(Command::new("git").args(&["stash", "pop"]).current_dir(absolute_path)); + } + } + + /// If any submodule has been initialized already, sync it unconditionally. + /// This avoids contributors checking in a submodule change by accident. + pub fn update_existing_submodules(&self) { + // Avoid running git when there isn't a git checkout. + if !self.config.submodules(&self.rust_info()) { + return; + } + let output = output( + self.config + .git() + .args(&["config", "--file"]) + .arg(&self.config.src.join(".gitmodules")) + .args(&["--get-regexp", "path"]), + ); + for line in output.lines() { + // Look for `submodule.$name.path = $path` + // Sample output: `submodule.src/rust-installer.path src/tools/rust-installer` + let submodule = Path::new(line.splitn(2, ' ').nth(1).unwrap()); + // Don't update the submodule unless it's already been cloned. + if GitInfo::new(false, submodule).is_managed_git_subrepository() { + self.update_submodule(submodule); + } + } + } + + /// Executes the entire build, as configured by the flags and configuration. + pub fn build(&mut self) { + unsafe { + crate::utils::job::setup(self); + } + + // Download rustfmt early so that it can be used in rust-analyzer configs. + let _ = &builder::Builder::new(&self).initial_rustfmt(); + + // hardcoded subcommands + match &self.config.cmd { + Subcommand::Format { check } => { + return core::build_steps::format::format( + &builder::Builder::new(&self), + *check, + &self.config.paths, + ); + } + Subcommand::Suggest { run } => { + return core::build_steps::suggest::suggest(&builder::Builder::new(&self), *run); + } + _ => (), + } + + { + let builder = builder::Builder::new(&self); + if let Some(path) = builder.paths.get(0) { + if path == Path::new("nonexistent/path/to/trigger/cargo/metadata") { + return; + } + } + } + + if !self.config.dry_run() { + { + self.config.dry_run = DryRun::SelfCheck; + let builder = builder::Builder::new(&self); + builder.execute_cli(); + } + self.config.dry_run = DryRun::Disabled; + let builder = builder::Builder::new(&self); + builder.execute_cli(); + } else { + let builder = builder::Builder::new(&self); + builder.execute_cli(); + } + + // Check for postponed failures from `test --no-fail-fast`. + let failures = self.delayed_failures.borrow(); + if failures.len() > 0 { + eprintln!("\n{} command(s) did not execute successfully:\n", failures.len()); + for failure in failures.iter() { + eprintln!(" - {failure}\n"); + } + exit!(1); + } + + #[cfg(feature = "build-metrics")] + self.metrics.persist(self); + } + + /// Clear out `dir` if `input` is newer. + /// + /// After this executes, it will also ensure that `dir` exists. + fn clear_if_dirty(&self, dir: &Path, input: &Path) -> bool { + let stamp = dir.join(".stamp"); + let mut cleared = false; + if mtime(&stamp) < mtime(input) { + self.verbose(&format!("Dirty - {}", dir.display())); + let _ = fs::remove_dir_all(dir); + cleared = true; + } else if stamp.exists() { + return cleared; + } + t!(fs::create_dir_all(dir)); + t!(File::create(stamp)); + cleared + } + + fn rust_info(&self) -> &GitInfo { + &self.config.rust_info + } + + /// Gets the space-separated set of activated features for the standard + /// library. + fn std_features(&self, target: TargetSelection) -> String { + let mut features = " panic-unwind".to_string(); + + match self.config.llvm_libunwind(target) { + LlvmLibunwind::InTree => features.push_str(" llvm-libunwind"), + LlvmLibunwind::System => features.push_str(" system-llvm-libunwind"), + LlvmLibunwind::No => {} + } + if self.config.backtrace { + features.push_str(" backtrace"); + } + if self.config.profiler_enabled(target) { + features.push_str(" profiler"); + } + features + } + + /// Gets the space-separated set of activated features for the compiler. + fn rustc_features(&self, kind: Kind) -> String { + let mut features = vec![]; + if self.config.jemalloc { + features.push("jemalloc"); + } + if self.config.llvm_enabled() || kind == Kind::Check { + features.push("llvm"); + } + // keep in sync with `bootstrap/compile.rs:rustc_cargo_env` + if self.config.rustc_parallel { + features.push("rustc_use_parallel_compiler"); + } + + // If debug logging is on, then we want the default for tracing: + // https://github.com/tokio-rs/tracing/blob/3dd5c03d907afdf2c39444a29931833335171554/tracing/src/level_filters.rs#L26 + // which is everything (including debug/trace/etc.) + // if its unset, if debug_assertions is on, then debug_logging will also be on + // as well as tracing *ignoring* this feature when debug_assertions is on + if !self.config.rust_debug_logging { + features.push("max_level_info"); + } + + features.join(" ") + } + + /// Component directory that Cargo will produce output into (e.g. + /// release/debug) + fn cargo_dir(&self) -> &'static str { + if self.config.rust_optimize.is_release() { "release" } else { "debug" } + } + + fn tools_dir(&self, compiler: Compiler) -> PathBuf { + let out = self + .out + .join(&*compiler.host.triple) + .join(format!("stage{}-tools-bin", compiler.stage)); + t!(fs::create_dir_all(&out)); + out + } + + /// Returns the root directory for all output generated in a particular + /// stage when running with a particular host compiler. + /// + /// The mode indicates what the root directory is for. + fn stage_out(&self, compiler: Compiler, mode: Mode) -> PathBuf { + let suffix = match mode { + Mode::Std => "-std", + Mode::Rustc => "-rustc", + Mode::Codegen => "-codegen", + Mode::ToolBootstrap => "-bootstrap-tools", + Mode::ToolStd | Mode::ToolRustc => "-tools", + }; + self.out.join(&*compiler.host.triple).join(format!("stage{}{}", compiler.stage, suffix)) + } + + /// Returns the root output directory for all Cargo output in a given stage, + /// running a particular compiler, whether or not we're building the + /// standard library, and targeting the specified architecture. + fn cargo_out(&self, compiler: Compiler, mode: Mode, target: TargetSelection) -> PathBuf { + self.stage_out(compiler, mode).join(&*target.triple).join(self.cargo_dir()) + } + + /// Root output directory for LLVM compiled for `target` + /// + /// Note that if LLVM is configured externally then the directory returned + /// will likely be empty. + fn llvm_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("llvm") + } + + fn lld_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("lld") + } + + /// Output directory for all documentation for a target + fn doc_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("doc") + } + + /// Output directory for all JSON-formatted documentation for a target + fn json_doc_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("json-doc") + } + + fn test_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("test") + } + + /// Output directory for all documentation for a target + fn compiler_doc_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("compiler-doc") + } + + /// Output directory for some generated md crate documentation for a target (temporary) + fn md_doc_out(&self, target: TargetSelection) -> Interned { + INTERNER.intern_path(self.out.join(&*target.triple).join("md-doc")) + } + + /// Returns `true` if no custom `llvm-config` is set for the specified target. + /// + /// If no custom `llvm-config` was specified then Rust's llvm will be used. + fn is_rust_llvm(&self, target: TargetSelection) -> bool { + match self.config.target_config.get(&target) { + Some(Target { llvm_has_rust_patches: Some(patched), .. }) => *patched, + Some(Target { llvm_config, .. }) => { + // If the user set llvm-config we assume Rust is not patched, + // but first check to see if it was configured by llvm-from-ci. + (self.config.llvm_from_ci && target == self.config.build) || llvm_config.is_none() + } + None => true, + } + } + + /// Returns the path to `FileCheck` binary for the specified target + fn llvm_filecheck(&self, target: TargetSelection) -> PathBuf { + let target_config = self.config.target_config.get(&target); + if let Some(s) = target_config.and_then(|c| c.llvm_filecheck.as_ref()) { + s.to_path_buf() + } else if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) { + let llvm_bindir = output(Command::new(s).arg("--bindir")); + let filecheck = Path::new(llvm_bindir.trim()).join(exe("FileCheck", target)); + if filecheck.exists() { + filecheck + } else { + // On Fedora the system LLVM installs FileCheck in the + // llvm subdirectory of the libdir. + let llvm_libdir = output(Command::new(s).arg("--libdir")); + let lib_filecheck = + Path::new(llvm_libdir.trim()).join("llvm").join(exe("FileCheck", target)); + if lib_filecheck.exists() { + lib_filecheck + } else { + // Return the most normal file name, even though + // it doesn't exist, so that any error message + // refers to that. + filecheck + } + } + } else { + let base = self.llvm_out(target).join("build"); + let base = if !self.ninja() && target.contains("msvc") { + if self.config.llvm_optimize { + if self.config.llvm_release_debuginfo { + base.join("RelWithDebInfo") + } else { + base.join("Release") + } + } else { + base.join("Debug") + } + } else { + base + }; + base.join("bin").join(exe("FileCheck", target)) + } + } + + /// Directory for libraries built from C/C++ code and shared between stages. + fn native_dir(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("native") + } + + /// Root output directory for rust_test_helpers library compiled for + /// `target` + fn test_helpers_out(&self, target: TargetSelection) -> PathBuf { + self.native_dir(target).join("rust-test-helpers") + } + + /// Adds the `RUST_TEST_THREADS` env var if necessary + fn add_rust_test_threads(&self, cmd: &mut Command) { + if env::var_os("RUST_TEST_THREADS").is_none() { + cmd.env("RUST_TEST_THREADS", self.jobs().to_string()); + } + } + + /// Returns the libdir of the snapshot compiler. + fn rustc_snapshot_libdir(&self) -> PathBuf { + self.rustc_snapshot_sysroot().join(libdir(self.config.build)) + } + + /// Returns the sysroot of the snapshot compiler. + fn rustc_snapshot_sysroot(&self) -> &Path { + static SYSROOT_CACHE: OnceCell = once_cell::sync::OnceCell::new(); + SYSROOT_CACHE.get_or_init(|| { + let mut rustc = Command::new(&self.initial_rustc); + rustc.args(&["--print", "sysroot"]); + output(&mut rustc).trim().into() + }) + } + + /// Runs a command, printing out nice contextual information if it fails. + fn run(&self, cmd: &mut Command) { + self.run_cmd(BootstrapCommand::from(cmd).fail_fast().output_mode( + match self.is_verbose() { + true => OutputMode::PrintAll, + false => OutputMode::PrintOutput, + }, + )); + } + + /// Runs a command, printing out contextual info if it fails, and delaying errors until the build finishes. + pub(crate) fn run_delaying_failure(&self, cmd: &mut Command) -> bool { + self.run_cmd(BootstrapCommand::from(cmd).delay_failure().output_mode( + match self.is_verbose() { + true => OutputMode::PrintAll, + false => OutputMode::PrintOutput, + }, + )) + } + + /// Runs a command, printing out nice contextual information if it fails. + fn run_quiet(&self, cmd: &mut Command) { + self.run_cmd( + BootstrapCommand::from(cmd).fail_fast().output_mode(OutputMode::SuppressOnSuccess), + ); + } + + /// Runs a command, printing out nice contextual information if it fails. + /// Exits if the command failed to execute at all, otherwise returns its + /// `status.success()`. + fn run_quiet_delaying_failure(&self, cmd: &mut Command) -> bool { + self.run_cmd( + BootstrapCommand::from(cmd).delay_failure().output_mode(OutputMode::SuppressOnSuccess), + ) + } + + /// A centralized function for running commands that do not return output. + pub(crate) fn run_cmd<'a, C: Into>>(&self, cmd: C) -> bool { + if self.config.dry_run() { + return true; + } + + let command = cmd.into(); + self.verbose(&format!("running: {command:?}")); + + let (output, print_error) = match command.output_mode { + mode @ (OutputMode::PrintAll | OutputMode::PrintOutput) => ( + command.command.status().map(|status| Output { + status, + stdout: Vec::new(), + stderr: Vec::new(), + }), + matches!(mode, OutputMode::PrintAll), + ), + OutputMode::SuppressOnSuccess => (command.command.output(), true), + }; + + let output = match output { + Ok(output) => output, + Err(e) => fail(&format!("failed to execute command: {:?}\nerror: {}", command, e)), + }; + let result = if !output.status.success() { + if print_error { + println!( + "\n\ncommand did not execute successfully: {:?}\n\ + expected success, got: {}\n\n\ + stdout ----\n{}\n\ + stderr ----\n{}\n\n", + command.command, + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + Err(()) + } else { + Ok(()) + }; + + match result { + Ok(_) => true, + Err(_) => { + match command.failure_behavior { + BehaviorOnFailure::DelayFail => { + if self.fail_fast { + exit!(1); + } + + let mut failures = self.delayed_failures.borrow_mut(); + failures.push(format!("{command:?}")); + } + BehaviorOnFailure::Exit => { + exit!(1); + } + BehaviorOnFailure::Ignore => {} + } + false + } + } + } + + pub fn is_verbose_than(&self, level: usize) -> bool { + self.verbosity > level + } + + /// Prints a message if this build is configured in more verbose mode than `level`. + fn verbose_than(&self, level: usize, msg: &str) { + if self.is_verbose_than(level) { + println!("{msg}"); + } + } + + fn info(&self, msg: &str) { + match self.config.dry_run { + DryRun::SelfCheck => (), + DryRun::Disabled | DryRun::UserSelected => { + println!("{msg}"); + } + } + } + + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] + fn msg_check( + &self, + what: impl Display, + target: impl Into>, + ) -> Option { + self.msg(Kind::Check, self.config.stage, what, self.config.build, target) + } + + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] + fn msg_doc( + &self, + compiler: Compiler, + what: impl Display, + target: impl Into> + Copy, + ) -> Option { + self.msg(Kind::Doc, compiler.stage, what, compiler.host, target.into()) + } + + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] + fn msg_build( + &self, + compiler: Compiler, + what: impl Display, + target: impl Into>, + ) -> Option { + self.msg(Kind::Build, compiler.stage, what, compiler.host, target) + } + + /// Return a `Group` guard for a [`Step`] that is built for each `--stage`. + /// + /// [`Step`]: crate::core::builder::Step + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] + fn msg( + &self, + action: impl Into, + stage: u32, + what: impl Display, + host: impl Into>, + target: impl Into>, + ) -> Option { + let action = action.into().description(); + let msg = |fmt| format!("{action} stage{stage} {what}{fmt}"); + let msg = if let Some(target) = target.into() { + let host = host.into().unwrap(); + if host == target { + msg(format_args!(" ({target})")) + } else { + msg(format_args!(" ({host} -> {target})")) + } + } else { + msg(format_args!("")) + }; + self.group(&msg) + } + + /// Return a `Group` guard for a [`Step`] that is only built once and isn't affected by `--stage`. + /// + /// [`Step`]: crate::core::builder::Step + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] + fn msg_unstaged( + &self, + action: impl Into, + what: impl Display, + target: TargetSelection, + ) -> Option { + let action = action.into().description(); + let msg = format!("{action} {what} for {target}"); + self.group(&msg) + } + + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] + fn msg_sysroot_tool( + &self, + action: impl Into, + stage: u32, + what: impl Display, + host: TargetSelection, + target: TargetSelection, + ) -> Option { + let action = action.into().description(); + let msg = |fmt| format!("{action} {what} {fmt}"); + let msg = if host == target { + msg(format_args!("(stage{stage} -> stage{}, {target})", stage + 1)) + } else { + msg(format_args!("(stage{stage}:{host} -> stage{}:{target})", stage + 1)) + }; + self.group(&msg) + } + + #[track_caller] + fn group(&self, msg: &str) -> Option { + match self.config.dry_run { + DryRun::SelfCheck => None, + DryRun::Disabled | DryRun::UserSelected => Some(gha::group(&msg)), + } + } + + /// Returns the number of parallel jobs that have been configured for this + /// build. + fn jobs(&self) -> u32 { + self.config.jobs.unwrap_or_else(|| { + std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32 + }) + } + + fn debuginfo_map_to(&self, which: GitRepo) -> Option { + if !self.config.rust_remap_debuginfo { + return None; + } + + match which { + GitRepo::Rustc => { + let sha = self.rust_sha().unwrap_or(&self.version); + Some(format!("/rustc/{sha}")) + } + GitRepo::Llvm => Some(String::from("/rustc/llvm")), + } + } + + /// Returns the path to the C compiler for the target specified. + fn cc(&self, target: TargetSelection) -> PathBuf { + if self.config.dry_run() { + return PathBuf::new(); + } + self.cc.borrow()[&target].path().into() + } + + /// Returns a list of flags to pass to the C compiler for the target + /// specified. + fn cflags(&self, target: TargetSelection, which: GitRepo, c: CLang) -> Vec { + if self.config.dry_run() { + return Vec::new(); + } + let base = match c { + CLang::C => self.cc.borrow()[&target].clone(), + CLang::Cxx => self.cxx.borrow()[&target].clone(), + }; + + // Filter out -O and /O (the optimization flags) that we picked up from + // cc-rs because the build scripts will determine that for themselves. + let mut base = base + .args() + .iter() + .map(|s| s.to_string_lossy().into_owned()) + .filter(|s| !s.starts_with("-O") && !s.starts_with("/O")) + .collect::>(); + + // If we're compiling C++ on macOS then we add a flag indicating that + // we want libc++ (more filled out than libstdc++), ensuring that + // LLVM/etc are all properly compiled. + if matches!(c, CLang::Cxx) && target.contains("apple-darwin") { + base.push("-stdlib=libc++".into()); + } + + // Work around an apparently bad MinGW / GCC optimization, + // See: https://lists.llvm.org/pipermail/cfe-dev/2016-December/051980.html + // See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=78936 + if &*target.triple == "i686-pc-windows-gnu" { + base.push("-fno-omit-frame-pointer".into()); + } + + if let Some(map_to) = self.debuginfo_map_to(which) { + let map = format!("{}={}", self.src.display(), map_to); + let cc = self.cc(target); + if cc.ends_with("clang") || cc.ends_with("gcc") { + base.push(format!("-fdebug-prefix-map={map}")); + } else if cc.ends_with("clang-cl.exe") { + base.push("-Xclang".into()); + base.push(format!("-fdebug-prefix-map={map}")); + } + } + base + } + + /// Returns the path to the `ar` archive utility for the target specified. + fn ar(&self, target: TargetSelection) -> Option { + if self.config.dry_run() { + return None; + } + self.ar.borrow().get(&target).cloned() + } + + /// Returns the path to the `ranlib` utility for the target specified. + fn ranlib(&self, target: TargetSelection) -> Option { + if self.config.dry_run() { + return None; + } + self.ranlib.borrow().get(&target).cloned() + } + + /// Returns the path to the C++ compiler for the target specified. + fn cxx(&self, target: TargetSelection) -> Result { + if self.config.dry_run() { + return Ok(PathBuf::new()); + } + match self.cxx.borrow().get(&target) { + Some(p) => Ok(p.path().into()), + None => Err(format!("target `{target}` is not configured as a host, only as a target")), + } + } + + /// Returns the path to the linker for the given target if it needs to be overridden. + fn linker(&self, target: TargetSelection) -> Option { + if self.config.dry_run() { + return Some(PathBuf::new()); + } + if let Some(linker) = self.config.target_config.get(&target).and_then(|c| c.linker.clone()) + { + Some(linker) + } else if target.contains("vxworks") { + // need to use CXX compiler as linker to resolve the exception functions + // that are only existed in CXX libraries + Some(self.cxx.borrow()[&target].path().into()) + } else if target != self.config.build + && helpers::use_host_linker(target) + && !target.contains("msvc") + { + Some(self.cc(target)) + } else if self.config.use_lld && !self.is_fuse_ld_lld(target) && self.build == target { + Some(self.initial_lld.clone()) + } else { + None + } + } + + // LLD is used through `-fuse-ld=lld` rather than directly. + // Only MSVC targets use LLD directly at the moment. + fn is_fuse_ld_lld(&self, target: TargetSelection) -> bool { + self.config.use_lld && !target.contains("msvc") + } + + fn lld_flags(&self, target: TargetSelection) -> impl Iterator { + let mut options = [None, None]; + + if self.config.use_lld { + if self.is_fuse_ld_lld(target) { + options[0] = Some("-Clink-arg=-fuse-ld=lld".to_string()); + } + + let no_threads = helpers::lld_flag_no_threads(target.contains("windows")); + options[1] = Some(format!("-Clink-arg=-Wl,{no_threads}")); + } + + IntoIterator::into_iter(options).flatten() + } + + /// Returns if this target should statically link the C runtime, if specified + fn crt_static(&self, target: TargetSelection) -> Option { + if target.contains("pc-windows-msvc") { + Some(true) + } else { + self.config.target_config.get(&target).and_then(|t| t.crt_static) + } + } + + /// Returns the "musl root" for this `target`, if defined + fn musl_root(&self, target: TargetSelection) -> Option<&Path> { + self.config + .target_config + .get(&target) + .and_then(|t| t.musl_root.as_ref()) + .or_else(|| self.config.musl_root.as_ref()) + .map(|p| &**p) + } + + /// Returns the "musl libdir" for this `target`. + fn musl_libdir(&self, target: TargetSelection) -> Option { + let t = self.config.target_config.get(&target)?; + if let libdir @ Some(_) = &t.musl_libdir { + return libdir.clone(); + } + self.musl_root(target).map(|root| root.join("lib")) + } + + /// Returns the sysroot for the wasi target, if defined + fn wasi_root(&self, target: TargetSelection) -> Option<&Path> { + self.config.target_config.get(&target).and_then(|t| t.wasi_root.as_ref()).map(|p| &**p) + } + + /// Returns `true` if this is a no-std `target`, if defined + fn no_std(&self, target: TargetSelection) -> Option { + self.config.target_config.get(&target).map(|t| t.no_std) + } + + /// Returns `true` if the target will be tested using the `remote-test-client` + /// and `remote-test-server` binaries. + fn remote_tested(&self, target: TargetSelection) -> bool { + self.qemu_rootfs(target).is_some() + || target.contains("android") + || env::var_os("TEST_DEVICE_ADDR").is_some() + } + + /// Returns the root of the "rootfs" image that this target will be using, + /// if one was configured. + /// + /// If `Some` is returned then that means that tests for this target are + /// emulated with QEMU and binaries will need to be shipped to the emulator. + fn qemu_rootfs(&self, target: TargetSelection) -> Option<&Path> { + self.config.target_config.get(&target).and_then(|t| t.qemu_rootfs.as_ref()).map(|p| &**p) + } + + /// Path to the python interpreter to use + fn python(&self) -> &Path { + if self.config.build.ends_with("apple-darwin") { + // Force /usr/bin/python3 on macOS for LLDB tests because we're loading the + // LLDB plugin's compiled module which only works with the system python + // (namely not Homebrew-installed python) + Path::new("/usr/bin/python3") + } else { + self.config + .python + .as_ref() + .expect("python is required for running LLDB or rustdoc tests") + } + } + + /// Temporary directory that extended error information is emitted to. + fn extended_error_dir(&self) -> PathBuf { + self.out.join("tmp/extended-error-metadata") + } + + /// Tests whether the `compiler` compiling for `target` should be forced to + /// use a stage1 compiler instead. + /// + /// Currently, by default, the build system does not perform a "full + /// bootstrap" by default where we compile the compiler three times. + /// Instead, we compile the compiler two times. The final stage (stage2) + /// just copies the libraries from the previous stage, which is what this + /// method detects. + /// + /// Here we return `true` if: + /// + /// * The build isn't performing a full bootstrap + /// * The `compiler` is in the final stage, 2 + /// * We're not cross-compiling, so the artifacts are already available in + /// stage1 + /// + /// When all of these conditions are met the build will lift artifacts from + /// the previous stage forward. + fn force_use_stage1(&self, stage: u32, target: TargetSelection) -> bool { + !self.config.full_bootstrap + && !self.config.download_rustc() + && stage >= 2 + && (self.hosts.iter().any(|h| *h == target) || target == self.build) + } + + /// Checks whether the `compiler` compiling for `target` should be forced to + /// use a stage2 compiler instead. + /// + /// When we download the pre-compiled version of rustc and compiler stage is >= 2, + /// it should be forced to use a stage2 compiler. + fn force_use_stage2(&self, stage: u32) -> bool { + self.config.download_rustc() && stage >= 2 + } + + /// Given `num` in the form "a.b.c" return a "release string" which + /// describes the release version number. + /// + /// For example on nightly this returns "a.b.c-nightly", on beta it returns + /// "a.b.c-beta.1" and on stable it just returns "a.b.c". + fn release(&self, num: &str) -> String { + match &self.config.channel[..] { + "stable" => num.to_string(), + "beta" => { + if !self.config.omit_git_hash { + format!("{}-beta.{}", num, self.beta_prerelease_version()) + } else { + format!("{num}-beta") + } + } + "nightly" => format!("{num}-nightly"), + _ => format!("{num}-dev"), + } + } + + fn beta_prerelease_version(&self) -> u32 { + fn extract_beta_rev_from_file>(version_file: P) -> Option { + let version = fs::read_to_string(version_file).ok()?; + + helpers::extract_beta_rev(&version) + } + + if let Some(s) = self.prerelease_version.get() { + return s; + } + + // First check if there is a version file available. + // If available, we read the beta revision from that file. + // This only happens when building from a source tarball when Git should not be used. + let count = extract_beta_rev_from_file(self.src.join("version")).unwrap_or_else(|| { + // Figure out how many merge commits happened since we branched off master. + // That's our beta number! + // (Note that we use a `..` range, not the `...` symmetric difference.) + output(self.config.git().arg("rev-list").arg("--count").arg("--merges").arg(format!( + "refs/remotes/origin/{}..HEAD", + self.config.stage0_metadata.config.nightly_branch + ))) + }); + let n = count.trim().parse().unwrap(); + self.prerelease_version.set(Some(n)); + n + } + + /// Returns the value of `release` above for Rust itself. + fn rust_release(&self) -> String { + self.release(&self.version) + } + + /// Returns the "package version" for a component given the `num` release + /// number. + /// + /// The package version is typically what shows up in the names of tarballs. + /// For channels like beta/nightly it's just the channel name, otherwise + /// it's the `num` provided. + fn package_vers(&self, num: &str) -> String { + match &self.config.channel[..] { + "stable" => num.to_string(), + "beta" => "beta".to_string(), + "nightly" => "nightly".to_string(), + _ => format!("{num}-dev"), + } + } + + /// Returns the value of `package_vers` above for Rust itself. + fn rust_package_vers(&self) -> String { + self.package_vers(&self.version) + } + + /// Returns the `version` string associated with this compiler for Rust + /// itself. + /// + /// Note that this is a descriptive string which includes the commit date, + /// sha, version, etc. + fn rust_version(&self) -> String { + let mut version = self.rust_info().version(self, &self.version); + if let Some(ref s) = self.config.description { + version.push_str(" ("); + version.push_str(s); + version.push(')'); + } + version + } + + /// Returns the full commit hash. + fn rust_sha(&self) -> Option<&str> { + self.rust_info().sha() + } + + /// Returns the `a.b.c` version that the given package is at. + fn release_num(&self, package: &str) -> String { + let toml_file_name = self.src.join(&format!("src/tools/{package}/Cargo.toml")); + let toml = t!(fs::read_to_string(&toml_file_name)); + for line in toml.lines() { + if let Some(stripped) = + line.strip_prefix("version = \"").and_then(|s| s.strip_suffix("\"")) + { + return stripped.to_owned(); + } + } + + panic!("failed to find version in {package}'s Cargo.toml") + } + + /// Returns `true` if unstable features should be enabled for the compiler + /// we're building. + fn unstable_features(&self) -> bool { + match &self.config.channel[..] { + "stable" | "beta" => false, + "nightly" | _ => true, + } + } + + /// Returns a Vec of all the dependencies of the given root crate, + /// including transitive dependencies and the root itself. Only includes + /// "local" crates (those in the local source tree, not from a registry). + fn in_tree_crates(&self, root: &str, target: Option) -> Vec<&Crate> { + let mut ret = Vec::new(); + let mut list = vec![INTERNER.intern_str(root)]; + let mut visited = HashSet::new(); + while let Some(krate) = list.pop() { + let krate = self + .crates + .get(&krate) + .unwrap_or_else(|| panic!("metadata missing for {krate}: {:?}", self.crates)); + ret.push(krate); + for dep in &krate.deps { + if !self.crates.contains_key(dep) { + // Ignore non-workspace members. + continue; + } + // Don't include optional deps if their features are not + // enabled. Ideally this would be computed from `cargo + // metadata --features …`, but that is somewhat slow. In + // the future, we may want to consider just filtering all + // build and dev dependencies in metadata::build. + if visited.insert(dep) + && (dep != "profiler_builtins" + || target + .map(|t| self.config.profiler_enabled(t)) + .unwrap_or_else(|| self.config.any_profiler_enabled())) + && (dep != "rustc_codegen_llvm" || self.config.llvm_enabled()) + { + list.push(*dep); + } + } + } + ret.sort_unstable_by_key(|krate| krate.name); // reproducible order needed for tests + ret + } + + fn read_stamp_file(&self, stamp: &Path) -> Vec<(PathBuf, DependencyType)> { + if self.config.dry_run() { + return Vec::new(); + } + + if !stamp.exists() { + eprintln!( + "ERROR: Unable to find the stamp file {}, did you try to keep a nonexistent build stage?", + stamp.display() + ); + crate::exit!(1); + } + + let mut paths = Vec::new(); + let contents = t!(fs::read(stamp), &stamp); + // This is the method we use for extracting paths from the stamp file passed to us. See + // run_cargo for more information (in compile.rs). + for part in contents.split(|b| *b == 0) { + if part.is_empty() { + continue; + } + let dependency_type = match part[0] as char { + 'h' => DependencyType::Host, + 's' => DependencyType::TargetSelfContained, + 't' => DependencyType::Target, + _ => unreachable!(), + }; + let path = PathBuf::from(t!(str::from_utf8(&part[1..]))); + paths.push((path, dependency_type)); + } + paths + } + + /// Copies a file from `src` to `dst` + pub fn copy(&self, src: &Path, dst: &Path) { + self.copy_internal(src, dst, false); + } + + fn copy_internal(&self, src: &Path, dst: &Path, dereference_symlinks: bool) { + if self.config.dry_run() { + return; + } + self.verbose_than(1, &format!("Copy {src:?} to {dst:?}")); + if src == dst { + return; + } + let _ = fs::remove_file(&dst); + let metadata = t!(src.symlink_metadata()); + let mut src = src.to_path_buf(); + if metadata.file_type().is_symlink() { + if dereference_symlinks { + src = t!(fs::canonicalize(src)); + } else { + let link = t!(fs::read_link(src)); + t!(self.symlink_file(link, dst)); + return; + } + } + if let Ok(()) = fs::hard_link(&src, dst) { + // Attempt to "easy copy" by creating a hard link + // (symlinks don't work on windows), but if that fails + // just fall back to a slow `copy` operation. + } else { + if let Err(e) = fs::copy(&src, dst) { + panic!("failed to copy `{}` to `{}`: {}", src.display(), dst.display(), e) + } + t!(fs::set_permissions(dst, metadata.permissions())); + let atime = FileTime::from_last_access_time(&metadata); + let mtime = FileTime::from_last_modification_time(&metadata); + t!(filetime::set_file_times(dst, atime, mtime)); + } + } + + /// Copies the `src` directory recursively to `dst`. Both are assumed to exist + /// when this function is called. + pub fn cp_r(&self, src: &Path, dst: &Path) { + if self.config.dry_run() { + return; + } + for f in self.read_dir(src) { + let path = f.path(); + let name = path.file_name().unwrap(); + let dst = dst.join(name); + if t!(f.file_type()).is_dir() { + t!(fs::create_dir_all(&dst)); + self.cp_r(&path, &dst); + } else { + let _ = fs::remove_file(&dst); + self.copy(&path, &dst); + } + } + } + + /// Copies the `src` directory recursively to `dst`. Both are assumed to exist + /// when this function is called. Unwanted files or directories can be skipped + /// by returning `false` from the filter function. + pub fn cp_filtered(&self, src: &Path, dst: &Path, filter: &dyn Fn(&Path) -> bool) { + // Immediately recurse with an empty relative path + self.recurse_(src, dst, Path::new(""), filter) + } + + // Inner function does the actual work + fn recurse_(&self, src: &Path, dst: &Path, relative: &Path, filter: &dyn Fn(&Path) -> bool) { + for f in self.read_dir(src) { + let path = f.path(); + let name = path.file_name().unwrap(); + let dst = dst.join(name); + let relative = relative.join(name); + // Only copy file or directory if the filter function returns true + if filter(&relative) { + if t!(f.file_type()).is_dir() { + let _ = fs::remove_dir_all(&dst); + self.create_dir(&dst); + self.recurse_(&path, &dst, &relative, filter); + } else { + let _ = fs::remove_file(&dst); + self.copy(&path, &dst); + } + } + } + } + + fn copy_to_folder(&self, src: &Path, dest_folder: &Path) { + let file_name = src.file_name().unwrap(); + let dest = dest_folder.join(file_name); + self.copy(src, &dest); + } + + fn install(&self, src: &Path, dstdir: &Path, perms: u32) { + if self.config.dry_run() { + return; + } + let dst = dstdir.join(src.file_name().unwrap()); + self.verbose_than(1, &format!("Install {src:?} to {dst:?}")); + t!(fs::create_dir_all(dstdir)); + if !src.exists() { + panic!("ERROR: File \"{}\" not found!", src.display()); + } + self.copy_internal(src, &dst, true); + chmod(&dst, perms); + } + + fn read(&self, path: &Path) -> String { + if self.config.dry_run() { + return String::new(); + } + t!(fs::read_to_string(path)) + } + + fn create_dir(&self, dir: &Path) { + if self.config.dry_run() { + return; + } + t!(fs::create_dir_all(dir)) + } + + fn remove_dir(&self, dir: &Path) { + if self.config.dry_run() { + return; + } + t!(fs::remove_dir_all(dir)) + } + + fn read_dir(&self, dir: &Path) -> impl Iterator { + let iter = match fs::read_dir(dir) { + Ok(v) => v, + Err(_) if self.config.dry_run() => return vec![].into_iter(), + Err(err) => panic!("could not read dir {dir:?}: {err:?}"), + }; + iter.map(|e| t!(e)).collect::>().into_iter() + } + + fn symlink_file, Q: AsRef>(&self, src: P, link: Q) -> io::Result<()> { + #[cfg(unix)] + use std::os::unix::fs::symlink as symlink_file; + #[cfg(windows)] + use std::os::windows::fs::symlink_file; + if !self.config.dry_run() { symlink_file(src.as_ref(), link.as_ref()) } else { Ok(()) } + } + + /// Returns if config.ninja is enabled, and checks for ninja existence, + /// exiting with a nicer error message if not. + fn ninja(&self) -> bool { + let mut cmd_finder = crate::core::sanity::Finder::new(); + + if self.config.ninja_in_file { + // Some Linux distros rename `ninja` to `ninja-build`. + // CMake can work with either binary name. + if cmd_finder.maybe_have("ninja-build").is_none() + && cmd_finder.maybe_have("ninja").is_none() + { + eprintln!( + " +Couldn't find required command: ninja (or ninja-build) + +You should install ninja as described at +, +or set `ninja = false` in the `[llvm]` section of `config.toml`. +Alternatively, set `download-ci-llvm = true` in that `[llvm]` section +to download LLVM rather than building it. +" + ); + exit!(1); + } + } + + // If ninja isn't enabled but we're building for MSVC then we try + // doubly hard to enable it. It was realized in #43767 that the msbuild + // CMake generator for MSVC doesn't respect configuration options like + // disabling LLVM assertions, which can often be quite important! + // + // In these cases we automatically enable Ninja if we find it in the + // environment. + if !self.config.ninja_in_file + && self.config.build.contains("msvc") + && cmd_finder.maybe_have("ninja").is_some() + { + return true; + } + + self.config.ninja_in_file + } + + pub fn colored_stdout R>(&self, f: F) -> R { + self.colored_stream_inner(StandardStream::stdout, self.config.stdout_is_tty, f) + } + + pub fn colored_stderr R>(&self, f: F) -> R { + self.colored_stream_inner(StandardStream::stderr, self.config.stderr_is_tty, f) + } + + fn colored_stream_inner(&self, constructor: C, is_tty: bool, f: F) -> R + where + C: Fn(ColorChoice) -> StandardStream, + F: FnOnce(&mut dyn WriteColor) -> R, + { + let choice = match self.config.color { + flags::Color::Always => ColorChoice::Always, + flags::Color::Never => ColorChoice::Never, + flags::Color::Auto if !is_tty => ColorChoice::Never, + flags::Color::Auto => ColorChoice::Auto, + }; + let mut stream = constructor(choice); + let result = f(&mut stream); + stream.reset().unwrap(); + result + } +} + +#[cfg(unix)] +fn chmod(path: &Path, perms: u32) { + use std::os::unix::fs::*; + t!(fs::set_permissions(path, fs::Permissions::from_mode(perms))); +} +#[cfg(windows)] +fn chmod(_path: &Path, _perms: u32) {} + +impl Compiler { + pub fn with_stage(mut self, stage: u32) -> Compiler { + self.stage = stage; + self + } + + /// Returns `true` if this is a snapshot compiler for `build`'s configuration + pub fn is_snapshot(&self, build: &Build) -> bool { + self.stage == 0 && self.host == build.build + } + + /// Returns if this compiler should be treated as a final stage one in the + /// current build session. + /// This takes into account whether we're performing a full bootstrap or + /// not; don't directly compare the stage with `2`! + pub fn is_final_stage(&self, build: &Build) -> bool { + let final_stage = if build.config.full_bootstrap { 2 } else { 1 }; + self.stage >= final_stage + } +} + +fn envify(s: &str) -> String { + s.chars() + .map(|c| match c { + '-' => '_', + c => c, + }) + .flat_map(|c| c.to_uppercase()) + .collect() +} + +pub fn find_recent_config_change_ids(current_id: usize) -> Vec { + if !CONFIG_CHANGE_HISTORY.contains(¤t_id) { + // If the current change-id is greater than the most recent one, return + // an empty list (it may be due to switching from a recent branch to an + // older one); otherwise, return the full list (assuming the user provided + // the incorrect change-id by accident). + if let Some(max_id) = CONFIG_CHANGE_HISTORY.iter().max() { + if ¤t_id > max_id { + return Vec::new(); + } + } + + return CONFIG_CHANGE_HISTORY.to_vec(); + } + + let index = CONFIG_CHANGE_HISTORY.iter().position(|&id| id == current_id).unwrap(); + + CONFIG_CHANGE_HISTORY + .iter() + .skip(index + 1) // Skip the current_id and IDs before it + .cloned() + .collect() +} diff --git a/src/bootstrap/src/tests/builder.rs b/src/bootstrap/src/tests/builder.rs new file mode 100644 index 000000000..96139f7b0 --- /dev/null +++ b/src/bootstrap/src/tests/builder.rs @@ -0,0 +1,701 @@ +use super::*; +use crate::core::config::{Config, DryRun, TargetSelection}; +use crate::core::build_steps::doc::DocumentationFormat; +use std::thread; + +fn configure(cmd: &str, host: &[&str], target: &[&str]) -> Config { + configure_with_args(&[cmd.to_owned()], host, target) +} + +fn configure_with_args(cmd: &[String], host: &[&str], target: &[&str]) -> Config { + let mut config = Config::parse(cmd); + // don't save toolstates + config.save_toolstates = None; + config.dry_run = DryRun::SelfCheck; + + // Ignore most submodules, since we don't need them for a dry run. + // But make sure to check out the `doc` and `rust-analyzer` submodules, since some steps need them + // just to know which commands to run. + let submodule_build = Build::new(Config { + // don't include LLVM, so CI doesn't require ninja/cmake to be installed + rust_codegen_backends: vec![], + ..Config::parse(&["check".to_owned()]) + }); + submodule_build.update_submodule(Path::new("src/doc/book")); + config.submodules = Some(false); + + config.ninja_in_file = false; + // try to avoid spurious failures in dist where we create/delete each others file + // HACK: rather than pull in `tempdir`, use the one that cargo has conveniently created for us + let dir = Path::new(env!("OUT_DIR")) + .join("tmp-rustbuild-tests") + .join(&thread::current().name().unwrap_or("unknown").replace(":", "-")); + t!(fs::create_dir_all(&dir)); + config.out = dir; + config.build = TargetSelection::from_user("A"); + config.hosts = host.iter().map(|s| TargetSelection::from_user(s)).collect(); + config.targets = target.iter().map(|s| TargetSelection::from_user(s)).collect(); + config +} + +fn first(v: Vec<(A, B)>) -> Vec { + v.into_iter().map(|(a, _)| a).collect::>() +} + +fn run_build(paths: &[PathBuf], config: Config) -> Cache { + let kind = config.cmd.kind(); + let build = Build::new(config); + let builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(kind), paths); + builder.cache +} + +fn check_cli(paths: [&str; N]) { + run_build( + &paths.map(PathBuf::from), + configure_with_args(&paths.map(String::from), &["A"], &["A"]), + ); +} + +macro_rules! std { + ($host:ident => $target:ident, stage = $stage:literal) => { + compile::Std::new( + Compiler { host: TargetSelection::from_user(stringify!($host)), stage: $stage }, + TargetSelection::from_user(stringify!($target)), + ) + }; +} + +macro_rules! doc_std { + ($host:ident => $target:ident, stage = $stage:literal) => {{ + let config = configure("doc", &["A"], &["A"]); + let build = Build::new(config); + let builder = Builder::new(&build); + doc::Std::new( + $stage, + TargetSelection::from_user(stringify!($target)), + &builder, + DocumentationFormat::HTML, + ) + }}; +} + +macro_rules! rustc { + ($host:ident => $target:ident, stage = $stage:literal) => { + compile::Rustc::new( + Compiler { host: TargetSelection::from_user(stringify!($host)), stage: $stage }, + TargetSelection::from_user(stringify!($target)), + ) + }; +} + +#[test] +fn test_valid() { + // make sure multi suite paths are accepted + check_cli(["test", "tests/ui/attr-start.rs", "tests/ui/attr-shebang.rs"]); +} + +#[test] +#[should_panic] +fn test_invalid() { + // make sure that invalid paths are caught, even when combined with valid paths + check_cli(["test", "library/std", "x"]); +} + +#[test] +fn test_intersection() { + let set = |paths: &[&str]| { + PathSet::Set(paths.into_iter().map(|p| TaskPath { path: p.into(), kind: None }).collect()) + }; + let library_set = set(&["library/core", "library/alloc", "library/std"]); + let mut command_paths = + vec![Path::new("library/core"), Path::new("library/alloc"), Path::new("library/stdarch")]; + let subset = library_set.intersection_removing_matches(&mut command_paths, Kind::Build); + assert_eq!(subset, set(&["library/core", "library/alloc"]),); + assert_eq!(command_paths, vec![Path::new("library/stdarch")]); +} + +#[test] +fn test_exclude() { + let mut config = configure("test", &["A"], &["A"]); + config.skip = vec!["src/tools/tidy".into()]; + let cache = run_build(&[], config); + + // Ensure we have really excluded tidy + assert!(!cache.contains::()); + + // Ensure other tests are not affected. + assert!(cache.contains::()); +} + +#[test] +fn test_exclude_kind() { + let path = PathBuf::from("compiler/rustc_data_structures"); + + let mut config = configure("test", &["A"], &["A"]); + // Ensure our test is valid, and `test::Rustc` would be run without the exclude. + assert!(run_build(&[], config.clone()).contains::()); + // Ensure tests for rustc are not skipped. + config.skip = vec![path.clone()]; + assert!(run_build(&[], config.clone()).contains::()); + // Ensure builds for rustc are not skipped. + assert!(run_build(&[], config).contains::()); +} + +/// Ensure that if someone passes both a single crate and `library`, all library crates get built. +#[test] +fn alias_and_path_for_library() { + let mut cache = + run_build(&["library".into(), "core".into()], configure("build", &["A"], &["A"])); + assert_eq!( + first(cache.all::()), + &[std!(A => A, stage = 0), std!(A => A, stage = 1)] + ); + + let mut cache = run_build(&["library".into(), "core".into()], configure("doc", &["A"], &["A"])); + assert_eq!(first(cache.all::()), &[doc_std!(A => A, stage = 0)]); +} + +#[test] +fn test_beta_rev_parsing() { + use crate::utils::helpers::extract_beta_rev; + + // single digit revision + assert_eq!(extract_beta_rev("1.99.9-beta.7 (xxxxxx)"), Some("7".to_string())); + // multiple digits + assert_eq!(extract_beta_rev("1.99.9-beta.777 (xxxxxx)"), Some("777".to_string())); + // nightly channel (no beta revision) + assert_eq!(extract_beta_rev("1.99.9-nightly (xxxxxx)"), None); + // stable channel (no beta revision) + assert_eq!(extract_beta_rev("1.99.9 (xxxxxxx)"), None); + // invalid string + assert_eq!(extract_beta_rev("invalid"), None); +} + +mod defaults { + use super::{configure, first, run_build}; + use crate::core::builder::*; + use crate::Config; + use pretty_assertions::assert_eq; + + #[test] + fn build_default() { + let mut cache = run_build(&[], configure("build", &["A"], &["A"])); + + let a = TargetSelection::from_user("A"); + assert_eq!( + first(cache.all::()), + &[std!(A => A, stage = 0), std!(A => A, stage = 1),] + ); + assert!(!cache.all::().is_empty()); + // Make sure rustdoc is only built once. + assert_eq!( + first(cache.all::()), + // Recall that rustdoc stages are off-by-one + // - this is the compiler it's _linked_ to, not built with. + &[tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }], + ); + assert_eq!(first(cache.all::()), &[rustc!(A => A, stage = 0)],); + } + + #[test] + fn build_stage_0() { + let config = Config { stage: 0, ..configure("build", &["A"], &["A"]) }; + let mut cache = run_build(&[], config); + + let a = TargetSelection::from_user("A"); + assert_eq!(first(cache.all::()), &[std!(A => A, stage = 0)]); + assert!(!cache.all::().is_empty()); + assert_eq!( + first(cache.all::()), + // This is the beta rustdoc. + // Add an assert here to make sure this is the only rustdoc built. + &[tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } }], + ); + assert!(cache.all::().is_empty()); + } + + #[test] + fn build_cross_compile() { + let config = Config { stage: 1, ..configure("build", &["A", "B"], &["A", "B"]) }; + let mut cache = run_build(&[], config); + + let a = TargetSelection::from_user("A"); + let b = TargetSelection::from_user("B"); + + // Ideally, this build wouldn't actually have `target: a` + // rustdoc/rustcc/std here (the user only requested a host=B build, so + // there's not really a need for us to build for target A in this case + // (since we're producing stage 1 libraries/binaries). But currently + // rustbuild is just a bit buggy here; this should be fixed though. + assert_eq!( + first(cache.all::()), + &[ + std!(A => A, stage = 0), + std!(A => A, stage = 1), + std!(A => B, stage = 0), + std!(A => B, stage = 1), + ] + ); + assert_eq!( + first(cache.all::()), + &[ + compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, + compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, + compile::Assemble { target_compiler: Compiler { host: b, stage: 1 } }, + ] + ); + assert_eq!( + first(cache.all::()), + &[ + tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }, + tool::Rustdoc { compiler: Compiler { host: b, stage: 1 } }, + ], + ); + assert_eq!( + first(cache.all::()), + &[rustc!(A => A, stage = 0), rustc!(A => B, stage = 0),] + ); + } + + #[test] + fn doc_default() { + let mut config = configure("doc", &["A"], &["A"]); + config.compiler_docs = true; + config.cmd = Subcommand::Doc { open: false, json: false }; + let mut cache = run_build(&[], config); + let a = TargetSelection::from_user("A"); + + // error_index_generator uses stage 0 to share rustdoc artifacts with the + // rustdoc tool. + assert_eq!(first(cache.all::()), &[doc::ErrorIndex { target: a },]); + assert_eq!( + first(cache.all::()), + &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 0 } }] + ); + // docs should be built with the beta compiler, not with the stage0 artifacts. + // recall that rustdoc is off-by-one: `stage` is the compiler rustdoc is _linked_ to, + // not the one it was built by. + assert_eq!( + first(cache.all::()), + &[tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } },] + ); + } +} + +mod dist { + use super::{first, run_build, Config}; + use crate::core::builder::*; + use pretty_assertions::assert_eq; + + fn configure(host: &[&str], target: &[&str]) -> Config { + Config { stage: 2, ..super::configure("dist", host, target) } + } + + #[test] + fn dist_baseline() { + let mut cache = run_build(&[], configure(&["A"], &["A"])); + + let a = TargetSelection::from_user("A"); + + assert_eq!(first(cache.all::()), &[dist::Docs { host: a },]); + assert_eq!(first(cache.all::()), &[dist::Mingw { host: a },]); + assert_eq!( + first(cache.all::()), + &[dist::Rustc { compiler: Compiler { host: a, stage: 2 } },] + ); + assert_eq!( + first(cache.all::()), + &[dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a },] + ); + assert_eq!(first(cache.all::()), &[dist::Src]); + // Make sure rustdoc is only built once. + assert_eq!( + first(cache.all::()), + &[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },] + ); + } + + #[test] + fn dist_with_targets() { + let mut cache = run_build(&[], configure(&["A"], &["A", "B"])); + + let a = TargetSelection::from_user("A"); + let b = TargetSelection::from_user("B"); + + assert_eq!( + first(cache.all::()), + &[dist::Docs { host: a }, dist::Docs { host: b },] + ); + assert_eq!( + first(cache.all::()), + &[dist::Mingw { host: a }, dist::Mingw { host: b },] + ); + assert_eq!( + first(cache.all::()), + &[dist::Rustc { compiler: Compiler { host: a, stage: 2 } },] + ); + assert_eq!( + first(cache.all::()), + &[ + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, + dist::Std { compiler: Compiler { host: a, stage: 2 }, target: b }, + ] + ); + assert_eq!(first(cache.all::()), &[dist::Src]); + } + + #[test] + fn dist_with_hosts() { + let mut cache = run_build(&[], configure(&["A", "B"], &["A", "B"])); + + let a = TargetSelection::from_user("A"); + let b = TargetSelection::from_user("B"); + + assert_eq!( + first(cache.all::()), + &[dist::Docs { host: a }, dist::Docs { host: b },] + ); + assert_eq!( + first(cache.all::()), + &[dist::Mingw { host: a }, dist::Mingw { host: b },] + ); + assert_eq!( + first(cache.all::()), + &[ + dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, + dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, + ] + ); + assert_eq!( + first(cache.all::()), + &[ + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, + ] + ); + assert_eq!( + first(cache.all::()), + &[ + std!(A => A, stage = 0), + std!(A => A, stage = 1), + std!(A => A, stage = 2), + std!(A => B, stage = 1), + std!(A => B, stage = 2), + ], + ); + assert_eq!(first(cache.all::()), &[dist::Src]); + } + + #[test] + fn dist_only_cross_host() { + let b = TargetSelection::from_user("B"); + let mut config = configure(&["A", "B"], &["A", "B"]); + config.docs = false; + config.extended = true; + config.hosts = vec![b]; + let mut cache = run_build(&[], config); + + assert_eq!( + first(cache.all::()), + &[dist::Rustc { compiler: Compiler { host: b, stage: 2 } },] + ); + assert_eq!( + first(cache.all::()), + &[rustc!(A => A, stage = 0), rustc!(A => B, stage = 1),] + ); + } + + #[test] + fn dist_with_targets_and_hosts() { + let mut cache = run_build(&[], configure(&["A", "B"], &["A", "B", "C"])); + + let a = TargetSelection::from_user("A"); + let b = TargetSelection::from_user("B"); + let c = TargetSelection::from_user("C"); + + assert_eq!( + first(cache.all::()), + &[dist::Docs { host: a }, dist::Docs { host: b }, dist::Docs { host: c },] + ); + assert_eq!( + first(cache.all::()), + &[dist::Mingw { host: a }, dist::Mingw { host: b }, dist::Mingw { host: c },] + ); + assert_eq!( + first(cache.all::()), + &[ + dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, + dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, + ] + ); + assert_eq!( + first(cache.all::()), + &[ + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, + dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c }, + ] + ); + assert_eq!(first(cache.all::()), &[dist::Src]); + } + + #[test] + fn dist_with_empty_host() { + let config = configure(&[], &["C"]); + let mut cache = run_build(&[], config); + + let a = TargetSelection::from_user("A"); + let c = TargetSelection::from_user("C"); + + assert_eq!(first(cache.all::()), &[dist::Docs { host: c },]); + assert_eq!(first(cache.all::()), &[dist::Mingw { host: c },]); + assert_eq!( + first(cache.all::()), + &[dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c },] + ); + } + + #[test] + fn dist_with_same_targets_and_hosts() { + let mut cache = run_build(&[], configure(&["A", "B"], &["A", "B"])); + + let a = TargetSelection::from_user("A"); + let b = TargetSelection::from_user("B"); + + assert_eq!( + first(cache.all::()), + &[dist::Docs { host: a }, dist::Docs { host: b },] + ); + assert_eq!( + first(cache.all::()), + &[dist::Mingw { host: a }, dist::Mingw { host: b },] + ); + assert_eq!( + first(cache.all::()), + &[ + dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, + dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, + ] + ); + assert_eq!( + first(cache.all::()), + &[ + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, + ] + ); + assert_eq!(first(cache.all::()), &[dist::Src]); + assert_eq!( + first(cache.all::()), + &[ + std!(A => A, stage = 0), + std!(A => A, stage = 1), + std!(A => A, stage = 2), + std!(A => B, stage = 1), + std!(A => B, stage = 2), + ] + ); + assert_eq!( + first(cache.all::()), + &[ + compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, + compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, + compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } }, + compile::Assemble { target_compiler: Compiler { host: b, stage: 2 } }, + ] + ); + } + + #[test] + fn build_all() { + let build = Build::new(configure(&["A", "B"], &["A", "B", "C"])); + let mut builder = Builder::new(&build); + builder.run_step_descriptions( + &Builder::get_step_descriptions(Kind::Build), + &["compiler/rustc".into(), "library".into()], + ); + + assert_eq!( + first(builder.cache.all::()), + &[ + std!(A => A, stage = 0), + std!(A => A, stage = 1), + std!(A => A, stage = 2), + std!(A => B, stage = 1), + std!(A => B, stage = 2), + std!(A => C, stage = 2), + ] + ); + assert_eq!(builder.cache.all::().len(), 5); + assert_eq!( + first(builder.cache.all::()), + &[ + rustc!(A => A, stage = 0), + rustc!(A => A, stage = 1), + rustc!(A => A, stage = 2), + rustc!(A => B, stage = 1), + rustc!(A => B, stage = 2), + ] + ); + } + + #[test] + fn build_with_empty_host() { + let config = configure(&[], &["C"]); + let build = Build::new(config); + let mut builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]); + + let a = TargetSelection::from_user("A"); + + assert_eq!( + first(builder.cache.all::()), + &[std!(A => A, stage = 0), std!(A => A, stage = 1), std!(A => C, stage = 2),] + ); + assert_eq!( + first(builder.cache.all::()), + &[ + compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, + compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, + compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } }, + ] + ); + assert_eq!( + first(builder.cache.all::()), + &[rustc!(A => A, stage = 0), rustc!(A => A, stage = 1),] + ); + } + + #[test] + fn test_with_no_doc_stage0() { + let mut config = configure(&["A"], &["A"]); + config.stage = 0; + config.paths = vec!["library/std".into()]; + config.cmd = Subcommand::Test { + test_args: vec![], + rustc_args: vec![], + no_fail_fast: false, + no_doc: true, + doc: false, + bless: false, + force_rerun: false, + compare_mode: None, + rustfix_coverage: false, + pass: None, + run: None, + only_modified: false, + skip: vec![], + extra_checks: None, + }; + + let build = Build::new(config); + let mut builder = Builder::new(&build); + + let host = TargetSelection::from_user("A"); + + builder.run_step_descriptions( + &[StepDescription::from::(Kind::Test)], + &["library/std".into()], + ); + + // Ensure we don't build any compiler artifacts. + assert!(!builder.cache.contains::()); + assert_eq!( + first(builder.cache.all::()), + &[test::Crate { + compiler: Compiler { host, stage: 0 }, + target: host, + mode: Mode::Std, + crates: vec![INTERNER.intern_str("std")], + },] + ); + } + + #[test] + fn doc_ci() { + let mut config = configure(&["A"], &["A"]); + config.compiler_docs = true; + config.cmd = Subcommand::Doc { open: false, json: false }; + let build = Build::new(config); + let mut builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Doc), &[]); + let a = TargetSelection::from_user("A"); + + // error_index_generator uses stage 1 to share rustdoc artifacts with the + // rustdoc tool. + assert_eq!( + first(builder.cache.all::()), + &[doc::ErrorIndex { target: a },] + ); + assert_eq!( + first(builder.cache.all::()), + &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 1 } }] + ); + // This is actually stage 1, but Rustdoc::run swaps out the compiler with + // stage minus 1 if --stage is not 0. Very confusing! + assert_eq!( + first(builder.cache.all::()), + &[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },] + ); + } + + #[test] + fn test_docs() { + // Behavior of `x.py test` doing various documentation tests. + let mut config = configure(&["A"], &["A"]); + config.cmd = Subcommand::Test { + test_args: vec![], + rustc_args: vec![], + no_fail_fast: false, + doc: true, + no_doc: false, + skip: vec![], + bless: false, + force_rerun: false, + compare_mode: None, + rustfix_coverage: false, + pass: None, + run: None, + only_modified: false, + extra_checks: None, + }; + // Make sure rustfmt binary not being found isn't an error. + config.channel = "beta".to_string(); + let build = Build::new(config); + let mut builder = Builder::new(&build); + + builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Test), &[]); + let a = TargetSelection::from_user("A"); + + // error_index_generator uses stage 1 to share rustdoc artifacts with the + // rustdoc tool. + assert_eq!( + first(builder.cache.all::()), + &[doc::ErrorIndex { target: a },] + ); + assert_eq!( + first(builder.cache.all::()), + &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 1 } }] + ); + // Unfortunately rustdoc is built twice. Once from stage1 for compiletest + // (and other things), and once from stage0 for std crates. Ideally it + // would only be built once. If someone wants to fix this, it might be + // worth investigating if it would be possible to test std from stage1. + // Note that the stages here are +1 than what they actually are because + // Rustdoc::run swaps out the compiler with stage minus 1 if --stage is + // not 0. + // + // The stage 0 copy is the one downloaded for bootstrapping. It is + // (currently) needed to run "cargo test" on the linkchecker, and + // should be relatively "free". + assert_eq!( + first(builder.cache.all::()), + &[ + tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } }, + tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }, + tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } }, + ] + ); + } +} diff --git a/src/bootstrap/src/tests/config.rs b/src/bootstrap/src/tests/config.rs new file mode 100644 index 000000000..59bd52a94 --- /dev/null +++ b/src/bootstrap/src/tests/config.rs @@ -0,0 +1,219 @@ +use crate::core::config::TomlConfig; +use super::{Config, Flags}; + +use clap::CommandFactory; +use serde::Deserialize; +use std::{ + env, + fs::{remove_file, File}, + io::Write, + path::Path, +}; + +fn parse(config: &str) -> Config { + Config::parse_inner(&["check".to_owned(), "--config=/does/not/exist".to_owned()], |&_| { + toml::from_str(config).unwrap() + }) +} + +#[test] +fn download_ci_llvm() { + if crate::core::build_steps::llvm::is_ci_llvm_modified(&parse("")) { + eprintln!("Detected LLVM as non-available: running in CI and modified LLVM in this change"); + return; + } + + let parse_llvm = |s| parse(s).llvm_from_ci; + let if_available = parse_llvm("llvm.download-ci-llvm = \"if-available\""); + + assert!(parse_llvm("llvm.download-ci-llvm = true")); + assert!(!parse_llvm("llvm.download-ci-llvm = false")); + assert_eq!(parse_llvm(""), if_available); + assert_eq!(parse_llvm("rust.channel = \"dev\""), if_available); + assert!(!parse_llvm("rust.channel = \"stable\"")); + assert!(parse_llvm("build.build = \"x86_64-unknown-linux-gnu\"")); + assert!(parse_llvm( + "llvm.assertions = true \r\n build.build = \"x86_64-unknown-linux-gnu\" \r\n llvm.download-ci-llvm = \"if-available\"" + )); + assert!(!parse_llvm( + "llvm.assertions = true \r\n build.build = \"aarch64-apple-darwin\" \r\n llvm.download-ci-llvm = \"if-available\"" + )); +} + +// FIXME(onur-ozkan): extend scope of the test +// refs: +// - https://github.com/rust-lang/rust/issues/109120 +// - https://github.com/rust-lang/rust/pull/109162#issuecomment-1496782487 +#[test] +fn detect_src_and_out() { + fn test(cfg: Config, build_dir: Option<&str>) { + // This will bring absolute form of `src/bootstrap` path + let current_dir = std::env::current_dir().unwrap(); + + // get `src` by moving into project root path + let expected_src = current_dir.ancestors().nth(2).unwrap(); + assert_eq!(&cfg.src, expected_src); + + // Sanity check for `src` + let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); + let expected_src = manifest_dir.ancestors().nth(2).unwrap(); + assert_eq!(&cfg.src, expected_src); + + // test if build-dir was manually given in config.toml + if let Some(custom_build_dir) = build_dir { + assert_eq!(&cfg.out, Path::new(custom_build_dir)); + } + // test the native bootstrap way + else { + // This should bring output path of bootstrap in absolute form + let cargo_target_dir = env::var_os("CARGO_TARGET_DIR").expect( + "CARGO_TARGET_DIR must been provided for the test environment from bootstrap", + ); + + // Move to `build` from `build/bootstrap` + let expected_out = Path::new(&cargo_target_dir).parent().unwrap(); + assert_eq!(&cfg.out, expected_out); + + let args: Vec = env::args().collect(); + + // Another test for `out` as a sanity check + // + // This will bring something similar to: + // `{build-dir}/bootstrap/debug/deps/bootstrap-c7ee91d5661e2804` + // `{build-dir}` can be anywhere, not just in the rust project directory. + let dep = Path::new(args.first().unwrap()); + let expected_out = dep.ancestors().nth(4).unwrap(); + + assert_eq!(&cfg.out, expected_out); + } + } + + test(parse(""), None); + + { + let build_dir = if cfg!(windows) { Some("C:\\tmp") } else { Some("/tmp") }; + test(parse("build.build-dir = \"/tmp\""), build_dir); + } +} + +#[test] +fn clap_verify() { + Flags::command().debug_assert(); +} + +#[test] +fn override_toml() { + let config = Config::parse_inner( + &[ + "check".to_owned(), + "--config=/does/not/exist".to_owned(), + "--set=change-id=1".to_owned(), + "--set=rust.lto=fat".to_owned(), + "--set=rust.deny-warnings=false".to_owned(), + "--set=build.gdb=\"bar\"".to_owned(), + "--set=build.tools=[\"cargo\"]".to_owned(), + "--set=llvm.build-config={\"foo\" = \"bar\"}".to_owned(), + ], + |&_| { + toml::from_str( + r#" +change-id = 0 +[rust] +lto = "off" +deny-warnings = true + +[build] +gdb = "foo" +tools = [] + +[llvm] +download-ci-llvm = false +build-config = {} + "#, + ) + .unwrap() + }, + ); + assert_eq!(config.change_id, Some(1), "setting top-level value"); + assert_eq!( + config.rust_lto, + crate::core::config::RustcLto::Fat, + "setting string value without quotes" + ); + assert_eq!(config.gdb, Some("bar".into()), "setting string value with quotes"); + assert!(!config.deny_warnings, "setting boolean value"); + assert_eq!( + config.tools, + Some(["cargo".to_string()].into_iter().collect()), + "setting list value" + ); + assert_eq!( + config.llvm_build_config, + [("foo".to_string(), "bar".to_string())].into_iter().collect(), + "setting dictionary value" + ); +} + +#[test] +#[should_panic] +fn override_toml_duplicate() { + Config::parse_inner( + &[ + "check".to_owned(), + "--config=/does/not/exist".to_owned(), + "--set=change-id=1".to_owned(), + "--set=change-id=2".to_owned(), + ], + |&_| toml::from_str("change-id = 0").unwrap(), + ); +} + +#[test] +fn profile_user_dist() { + fn get_toml(file: &Path) -> TomlConfig { + let contents = if file.ends_with("config.toml") { + "profile = \"user\"".to_owned() + } else { + assert!(file.ends_with("config.dist.toml")); + std::fs::read_to_string(file).unwrap() + }; + toml::from_str(&contents) + .and_then(|table: toml::Value| TomlConfig::deserialize(table)) + .unwrap() + } + Config::parse_inner(&["check".to_owned()], get_toml); +} + +#[test] +fn rust_optimize() { + assert!(parse("").rust_optimize.is_release()); + assert!(!parse("rust.optimize = false").rust_optimize.is_release()); + assert!(parse("rust.optimize = true").rust_optimize.is_release()); + assert!(!parse("rust.optimize = 0").rust_optimize.is_release()); + assert!(parse("rust.optimize = 1").rust_optimize.is_release()); + assert!(parse("rust.optimize = \"s\"").rust_optimize.is_release()); + assert_eq!(parse("rust.optimize = 1").rust_optimize.get_opt_level(), Some("1".to_string())); + assert_eq!(parse("rust.optimize = \"s\"").rust_optimize.get_opt_level(), Some("s".to_string())); +} + +#[test] +#[should_panic] +fn invalid_rust_optimize() { + parse("rust.optimize = \"a\""); +} + +#[test] +fn verify_file_integrity() { + let config = parse(""); + + let tempfile = config.tempdir().join(".tmp-test-file"); + File::create(&tempfile).unwrap().write_all(b"dummy value").unwrap(); + assert!(tempfile.exists()); + + assert!( + config + .verify(&tempfile, "7e255dd9542648a8779268a0f268b891a198e9828e860ed23f826440e786eae5") + ); + + remove_file(tempfile).unwrap(); +} diff --git a/src/bootstrap/src/tests/setup.rs b/src/bootstrap/src/tests/setup.rs new file mode 100644 index 000000000..0fe6e4a46 --- /dev/null +++ b/src/bootstrap/src/tests/setup.rs @@ -0,0 +1,14 @@ +use super::{RUST_ANALYZER_SETTINGS, SETTINGS_HASHES}; +use sha2::Digest; + +#[test] +fn check_matching_settings_hash() { + let mut hasher = sha2::Sha256::new(); + hasher.update(&RUST_ANALYZER_SETTINGS); + let hash = hex::encode(hasher.finalize().as_slice()); + assert_eq!( + &hash, + SETTINGS_HASHES.last().unwrap(), + "Update `SETTINGS_HASHES` with the new hash of `src/etc/rust_analyzer_settings.json`" + ); +} diff --git a/src/bootstrap/src/utils/bin_helpers.rs b/src/bootstrap/src/utils/bin_helpers.rs new file mode 100644 index 000000000..c90fd2805 --- /dev/null +++ b/src/bootstrap/src/utils/bin_helpers.rs @@ -0,0 +1,28 @@ +//! This file is meant to be included directly from bootstrap shims to avoid a +//! dependency on the bootstrap library. This reduces the binary size and +//! improves compilation time by reducing the linking time. + +/// Parses the value of the "RUSTC_VERBOSE" environment variable and returns it as a `usize`. +/// If it was not defined, returns 0 by default. +/// +/// Panics if "RUSTC_VERBOSE" is defined with the value that is not an unsigned integer. +pub(crate) fn parse_rustc_verbose() -> usize { + use std::str::FromStr; + + match std::env::var("RUSTC_VERBOSE") { + Ok(s) => usize::from_str(&s).expect("RUSTC_VERBOSE should be an integer"), + Err(_) => 0, + } +} + +/// Parses the value of the "RUSTC_STAGE" environment variable and returns it as a `String`. +/// +/// If "RUSTC_STAGE" was not set, the program will be terminated with 101. +pub(crate) fn parse_rustc_stage() -> String { + std::env::var("RUSTC_STAGE").unwrap_or_else(|_| { + // Don't panic here; it's reasonable to try and run these shims directly. Give a helpful error instead. + eprintln!("rustc shim: FATAL: RUSTC_STAGE was not set"); + eprintln!("rustc shim: NOTE: use `x.py build -vvv` to see all environment variables set by bootstrap"); + std::process::exit(101); + }) +} diff --git a/src/bootstrap/src/utils/cache.rs b/src/bootstrap/src/utils/cache.rs new file mode 100644 index 000000000..1b2aa9c23 --- /dev/null +++ b/src/bootstrap/src/utils/cache.rs @@ -0,0 +1,272 @@ +use std::any::{Any, TypeId}; +use std::borrow::Borrow; +use std::cell::RefCell; +use std::cmp::Ordering; +use std::collections::HashMap; +use std::fmt; +use std::hash::{Hash, Hasher}; +use std::marker::PhantomData; +use std::mem; +use std::ops::Deref; +use std::path::PathBuf; +use std::sync::Mutex; + +// FIXME: replace with std::lazy after it gets stabilized and reaches beta +use once_cell::sync::Lazy; + +use crate::core::builder::Step; + +pub struct Interned(usize, PhantomData<*const T>); + +impl Default for Interned { + fn default() -> Self { + T::default().intern() + } +} + +impl Copy for Interned {} +impl Clone for Interned { + fn clone(&self) -> Interned { + *self + } +} + +impl PartialEq for Interned { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } +} +impl Eq for Interned {} + +impl PartialEq for Interned { + fn eq(&self, other: &str) -> bool { + *self == other + } +} +impl<'a> PartialEq<&'a str> for Interned { + fn eq(&self, other: &&str) -> bool { + **self == **other + } +} +impl<'a, T> PartialEq<&'a Interned> for Interned { + fn eq(&self, other: &&Self) -> bool { + self.0 == other.0 + } +} +impl<'a, T> PartialEq> for &'a Interned { + fn eq(&self, other: &Interned) -> bool { + self.0 == other.0 + } +} + +unsafe impl Send for Interned {} +unsafe impl Sync for Interned {} + +impl fmt::Display for Interned { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s: &str = &*self; + f.write_str(s) + } +} + +impl fmt::Debug for Interned +where + Self: Deref, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s: &U = &*self; + f.write_fmt(format_args!("{s:?}")) + } +} + +impl Hash for Interned { + fn hash(&self, state: &mut H) { + let l = T::intern_cache().lock().unwrap(); + l.get(*self).hash(state) + } +} + +impl Deref for Interned { + type Target = T::Target; + fn deref(&self) -> &Self::Target { + let l = T::intern_cache().lock().unwrap(); + unsafe { mem::transmute::<&Self::Target, &Self::Target>(l.get(*self)) } + } +} + +impl, U: ?Sized> AsRef for Interned { + fn as_ref(&self) -> &U { + let l = T::intern_cache().lock().unwrap(); + unsafe { mem::transmute::<&U, &U>(l.get(*self).as_ref()) } + } +} + +impl PartialOrd for Interned { + fn partial_cmp(&self, other: &Self) -> Option { + let l = T::intern_cache().lock().unwrap(); + l.get(*self).partial_cmp(l.get(*other)) + } +} + +impl Ord for Interned { + fn cmp(&self, other: &Self) -> Ordering { + let l = T::intern_cache().lock().unwrap(); + l.get(*self).cmp(l.get(*other)) + } +} + +struct TyIntern { + items: Vec, + set: HashMap>, +} + +impl Default for TyIntern { + fn default() -> Self { + TyIntern { items: Vec::new(), set: Default::default() } + } +} + +impl TyIntern { + fn intern_borrow(&mut self, item: &B) -> Interned + where + B: Eq + Hash + ToOwned + ?Sized, + T: Borrow, + { + if let Some(i) = self.set.get(&item) { + return *i; + } + let item = item.to_owned(); + let interned = Interned(self.items.len(), PhantomData::<*const T>); + self.set.insert(item.clone(), interned); + self.items.push(item); + interned + } + + fn intern(&mut self, item: T) -> Interned { + if let Some(i) = self.set.get(&item) { + return *i; + } + let interned = Interned(self.items.len(), PhantomData::<*const T>); + self.set.insert(item.clone(), interned); + self.items.push(item); + interned + } + + fn get(&self, i: Interned) -> &T { + &self.items[i.0] + } +} + +#[derive(Default)] +pub struct Interner { + strs: Mutex>, + paths: Mutex>, + lists: Mutex>>, +} + +trait Internable: Clone + Eq + Hash + 'static { + fn intern_cache() -> &'static Mutex>; + + fn intern(self) -> Interned { + Self::intern_cache().lock().unwrap().intern(self) + } +} + +impl Internable for String { + fn intern_cache() -> &'static Mutex> { + &INTERNER.strs + } +} + +impl Internable for PathBuf { + fn intern_cache() -> &'static Mutex> { + &INTERNER.paths + } +} + +impl Internable for Vec { + fn intern_cache() -> &'static Mutex> { + &INTERNER.lists + } +} + +impl Interner { + pub fn intern_str(&self, s: &str) -> Interned { + self.strs.lock().unwrap().intern_borrow(s) + } + pub fn intern_string(&self, s: String) -> Interned { + self.strs.lock().unwrap().intern(s) + } + + pub fn intern_path(&self, s: PathBuf) -> Interned { + self.paths.lock().unwrap().intern(s) + } + + pub fn intern_list(&self, v: Vec) -> Interned> { + self.lists.lock().unwrap().intern(v) + } +} + +pub static INTERNER: Lazy = Lazy::new(Interner::default); + +/// This is essentially a `HashMap` which allows storing any type in its input and +/// any type in its output. It is a write-once cache; values are never evicted, +/// which means that references to the value can safely be returned from the +/// `get()` method. +#[derive(Debug)] +pub struct Cache( + RefCell< + HashMap< + TypeId, + Box, // actually a HashMap> + >, + >, +); + +impl Cache { + pub fn new() -> Cache { + Cache(RefCell::new(HashMap::new())) + } + + pub fn put(&self, step: S, value: S::Output) { + let mut cache = self.0.borrow_mut(); + let type_id = TypeId::of::(); + let stepcache = cache + .entry(type_id) + .or_insert_with(|| Box::new(HashMap::::new())) + .downcast_mut::>() + .expect("invalid type mapped"); + assert!(!stepcache.contains_key(&step), "processing {step:?} a second time"); + stepcache.insert(step, value); + } + + pub fn get(&self, step: &S) -> Option { + let mut cache = self.0.borrow_mut(); + let type_id = TypeId::of::(); + let stepcache = cache + .entry(type_id) + .or_insert_with(|| Box::new(HashMap::::new())) + .downcast_mut::>() + .expect("invalid type mapped"); + stepcache.get(step).cloned() + } +} + +#[cfg(test)] +impl Cache { + pub fn all(&mut self) -> Vec<(S, S::Output)> { + let cache = self.0.get_mut(); + let type_id = TypeId::of::(); + let mut v = cache + .remove(&type_id) + .map(|b| b.downcast::>().expect("correct type")) + .map(|m| m.into_iter().collect::>()) + .unwrap_or_default(); + v.sort_by_key(|(s, _)| s.clone()); + v + } + + pub fn contains(&self) -> bool { + self.0.borrow().contains_key(&TypeId::of::()) + } +} diff --git a/src/bootstrap/src/utils/cc_detect.rs b/src/bootstrap/src/utils/cc_detect.rs new file mode 100644 index 000000000..52b36ce75 --- /dev/null +++ b/src/bootstrap/src/utils/cc_detect.rs @@ -0,0 +1,288 @@ +//! C-compiler probing and detection. +//! +//! This module will fill out the `cc` and `cxx` maps of `Build` by looking for +//! C and C++ compilers for each target configured. A compiler is found through +//! a number of vectors (in order of precedence) +//! +//! 1. Configuration via `target.$target.cc` in `config.toml`. +//! 2. Configuration via `target.$target.android-ndk` in `config.toml`, if +//! applicable +//! 3. Special logic to probe on OpenBSD +//! 4. The `CC_$target` environment variable. +//! 5. The `CC` environment variable. +//! 6. "cc" +//! +//! Some of this logic is implemented here, but much of it is farmed out to the +//! `cc` crate itself, so we end up having the same fallbacks as there. +//! Similar logic is then used to find a C++ compiler, just some s/cc/c++/ is +//! used. +//! +//! It is intended that after this module has run no C/C++ compiler will +//! ever be probed for. Instead the compilers found here will be used for +//! everything. + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::{env, iter}; + +use crate::core::config::TargetSelection; +use crate::utils::helpers::output; +use crate::{Build, CLang, GitRepo}; + +// The `cc` crate doesn't provide a way to obtain a path to the detected archiver, +// so use some simplified logic here. First we respect the environment variable `AR`, then +// try to infer the archiver path from the C compiler path. +// In the future this logic should be replaced by calling into the `cc` crate. +fn cc2ar(cc: &Path, target: TargetSelection) -> Option { + if let Some(ar) = env::var_os(format!("AR_{}", target.triple.replace("-", "_"))) { + Some(PathBuf::from(ar)) + } else if let Some(ar) = env::var_os("AR") { + Some(PathBuf::from(ar)) + } else if target.contains("msvc") { + None + } else if target.contains("musl") { + Some(PathBuf::from("ar")) + } else if target.contains("openbsd") { + Some(PathBuf::from("ar")) + } else if target.contains("vxworks") { + Some(PathBuf::from("wr-ar")) + } else if target.contains("android") { + Some(cc.parent().unwrap().join(PathBuf::from("llvm-ar"))) + } else { + let parent = cc.parent().unwrap(); + let file = cc.file_name().unwrap().to_str().unwrap(); + for suffix in &["gcc", "cc", "clang"] { + if let Some(idx) = file.rfind(suffix) { + let mut file = file[..idx].to_owned(); + file.push_str("ar"); + return Some(parent.join(&file)); + } + } + Some(parent.join(file)) + } +} + +fn new_cc_build(build: &Build, target: TargetSelection) -> cc::Build { + let mut cfg = cc::Build::new(); + cfg.cargo_metadata(false) + .opt_level(2) + .warnings(false) + .debug(false) + // Compress debuginfo + .flag_if_supported("-gz") + .target(&target.triple) + .host(&build.build.triple); + match build.crt_static(target) { + Some(a) => { + cfg.static_crt(a); + } + None => { + if target.contains("msvc") { + cfg.static_crt(true); + } + if target.contains("musl") { + cfg.static_flag(true); + } + } + } + cfg +} + +pub fn find(build: &Build) { + // For all targets we're going to need a C compiler for building some shims + // and such as well as for being a linker for Rust code. + let targets = build + .targets + .iter() + .chain(&build.hosts) + .cloned() + .chain(iter::once(build.build)) + .collect::>(); + for target in targets.into_iter() { + find_target(build, target); + } +} + +pub fn find_target(build: &Build, target: TargetSelection) { + let mut cfg = new_cc_build(build, target); + let config = build.config.target_config.get(&target); + if let Some(cc) = config + .and_then(|c| c.cc.clone()) + .or_else(|| default_compiler(&mut cfg, Language::C, target, build)) + { + cfg.compiler(cc); + } + + let compiler = cfg.get_compiler(); + let ar = if let ar @ Some(..) = config.and_then(|c| c.ar.clone()) { + ar + } else { + cc2ar(compiler.path(), target) + }; + + build.cc.borrow_mut().insert(target, compiler.clone()); + let cflags = build.cflags(target, GitRepo::Rustc, CLang::C); + + // If we use llvm-libunwind, we will need a C++ compiler as well for all targets + // We'll need one anyways if the target triple is also a host triple + let mut cfg = new_cc_build(build, target); + cfg.cpp(true); + let cxx_configured = if let Some(cxx) = config + .and_then(|c| c.cxx.clone()) + .or_else(|| default_compiler(&mut cfg, Language::CPlusPlus, target, build)) + { + cfg.compiler(cxx); + true + } else { + // Use an auto-detected compiler (or one configured via `CXX_target_triple` env vars). + cfg.try_get_compiler().is_ok() + }; + + // for VxWorks, record CXX compiler which will be used in lib.rs:linker() + if cxx_configured || target.contains("vxworks") { + let compiler = cfg.get_compiler(); + build.cxx.borrow_mut().insert(target, compiler); + } + + build.verbose(&format!("CC_{} = {:?}", &target.triple, build.cc(target))); + build.verbose(&format!("CFLAGS_{} = {:?}", &target.triple, cflags)); + if let Ok(cxx) = build.cxx(target) { + let cxxflags = build.cflags(target, GitRepo::Rustc, CLang::Cxx); + build.verbose(&format!("CXX_{} = {:?}", &target.triple, cxx)); + build.verbose(&format!("CXXFLAGS_{} = {:?}", &target.triple, cxxflags)); + } + if let Some(ar) = ar { + build.verbose(&format!("AR_{} = {:?}", &target.triple, ar)); + build.ar.borrow_mut().insert(target, ar); + } + + if let Some(ranlib) = config.and_then(|c| c.ranlib.clone()) { + build.ranlib.borrow_mut().insert(target, ranlib); + } +} + +fn default_compiler( + cfg: &mut cc::Build, + compiler: Language, + target: TargetSelection, + build: &Build, +) -> Option { + match &*target.triple { + // When compiling for android we may have the NDK configured in the + // config.toml in which case we look there. Otherwise the default + // compiler already takes into account the triple in question. + t if t.contains("android") => build + .config + .android_ndk + .as_ref() + .map(|ndk| ndk_compiler(compiler, &*target.triple, ndk)), + + // The default gcc version from OpenBSD may be too old, try using egcc, + // which is a gcc version from ports, if this is the case. + t if t.contains("openbsd") => { + let c = cfg.get_compiler(); + let gnu_compiler = compiler.gcc(); + if !c.path().ends_with(gnu_compiler) { + return None; + } + + let output = output(c.to_command().arg("--version")); + let i = output.find(" 4.")?; + match output[i + 3..].chars().next().unwrap() { + '0'..='6' => {} + _ => return None, + } + let alternative = format!("e{gnu_compiler}"); + if Command::new(&alternative).output().is_ok() { + Some(PathBuf::from(alternative)) + } else { + None + } + } + + "mips-unknown-linux-musl" if compiler == Language::C => { + if cfg.get_compiler().path().to_str() == Some("gcc") { + Some(PathBuf::from("mips-linux-musl-gcc")) + } else { + None + } + } + "mipsel-unknown-linux-musl" if compiler == Language::C => { + if cfg.get_compiler().path().to_str() == Some("gcc") { + Some(PathBuf::from("mipsel-linux-musl-gcc")) + } else { + None + } + } + + t if t.contains("musl") && compiler == Language::C => { + if let Some(root) = build.musl_root(target) { + let guess = root.join("bin/musl-gcc"); + if guess.exists() { Some(guess) } else { None } + } else { + None + } + } + + _ => None, + } +} + +pub(crate) fn ndk_compiler(compiler: Language, triple: &str, ndk: &Path) -> PathBuf { + let mut triple_iter = triple.split("-"); + let triple_translated = if let Some(arch) = triple_iter.next() { + let arch_new = match arch { + "arm" | "armv7" | "armv7neon" | "thumbv7" | "thumbv7neon" => "armv7a", + other => other, + }; + std::iter::once(arch_new).chain(triple_iter).collect::>().join("-") + } else { + triple.to_string() + }; + + // API 19 is the earliest API level supported by NDK r25b but AArch64 and x86_64 support + // begins at API level 21. + let api_level = + if triple.contains("aarch64") || triple.contains("x86_64") { "21" } else { "19" }; + let compiler = format!("{}{}-{}", triple_translated, api_level, compiler.clang()); + let host_tag = if cfg!(target_os = "macos") { + // The NDK uses universal binaries, so this is correct even on ARM. + "darwin-x86_64" + } else if cfg!(target_os = "windows") { + "windows-x86_64" + } else { + // NDK r25b only has official releases for macOS, Windows and Linux. + // Try the Linux directory everywhere else, on the assumption that the OS has an + // emulation layer that can cope (e.g. BSDs). + "linux-x86_64" + }; + ndk.join("toolchains").join("llvm").join("prebuilt").join(host_tag).join("bin").join(compiler) +} + +/// The target programming language for a native compiler. +#[derive(PartialEq)] +pub(crate) enum Language { + /// The compiler is targeting C. + C, + /// The compiler is targeting C++. + CPlusPlus, +} + +impl Language { + /// Obtains the name of a compiler in the GCC collection. + fn gcc(self) -> &'static str { + match self { + Language::C => "gcc", + Language::CPlusPlus => "g++", + } + } + + /// Obtains the name of a compiler in the clang suite. + fn clang(self) -> &'static str { + match self { + Language::C => "clang", + Language::CPlusPlus => "clang++", + } + } +} diff --git a/src/bootstrap/src/utils/channel.rs b/src/bootstrap/src/utils/channel.rs new file mode 100644 index 000000000..e59d7f22a --- /dev/null +++ b/src/bootstrap/src/utils/channel.rs @@ -0,0 +1,159 @@ +//! Build configuration for Rust's release channels. +//! +//! Implements the stable/beta/nightly channel distinctions by setting various +//! flags like the `unstable_features`, calculating variables like `release` and +//! `package_vers`, and otherwise indicating to the compiler what it should +//! print out as part of its version information. + +use std::fs; +use std::path::Path; +use std::process::Command; + +use crate::utils::helpers::{output, t}; +use crate::Build; + +#[derive(Clone, Default)] +pub enum GitInfo { + /// This is not a git repository. + #[default] + Absent, + /// This is a git repository. + /// If the info should be used (`omit_git_hash` is false), this will be + /// `Some`, otherwise it will be `None`. + Present(Option), + /// This is not a git repository, but the info can be fetched from the + /// `git-commit-info` file. + RecordedForTarball(Info), +} + +#[derive(Clone)] +pub struct Info { + pub commit_date: String, + pub sha: String, + pub short_sha: String, +} + +impl GitInfo { + pub fn new(omit_git_hash: bool, dir: &Path) -> GitInfo { + // See if this even begins to look like a git dir + if !dir.join(".git").exists() { + match read_commit_info_file(dir) { + Some(info) => return GitInfo::RecordedForTarball(info), + None => return GitInfo::Absent, + } + } + + // Make sure git commands work + match Command::new("git").arg("rev-parse").current_dir(dir).output() { + Ok(ref out) if out.status.success() => {} + _ => return GitInfo::Absent, + } + + // If we're ignoring the git info, we don't actually need to collect it, just make sure this + // was a git repo in the first place. + if omit_git_hash { + return GitInfo::Present(None); + } + + // Ok, let's scrape some info + let ver_date = output( + Command::new("git") + .current_dir(dir) + .arg("log") + .arg("-1") + .arg("--date=short") + .arg("--pretty=format:%cd"), + ); + let ver_hash = output(Command::new("git").current_dir(dir).arg("rev-parse").arg("HEAD")); + let short_ver_hash = output( + Command::new("git").current_dir(dir).arg("rev-parse").arg("--short=9").arg("HEAD"), + ); + GitInfo::Present(Some(Info { + commit_date: ver_date.trim().to_string(), + sha: ver_hash.trim().to_string(), + short_sha: short_ver_hash.trim().to_string(), + })) + } + + pub fn info(&self) -> Option<&Info> { + match self { + GitInfo::Absent => None, + GitInfo::Present(info) => info.as_ref(), + GitInfo::RecordedForTarball(info) => Some(info), + } + } + + pub fn sha(&self) -> Option<&str> { + self.info().map(|s| &s.sha[..]) + } + + pub fn sha_short(&self) -> Option<&str> { + self.info().map(|s| &s.short_sha[..]) + } + + pub fn commit_date(&self) -> Option<&str> { + self.info().map(|s| &s.commit_date[..]) + } + + pub fn version(&self, build: &Build, num: &str) -> String { + let mut version = build.release(num); + if let Some(ref inner) = self.info() { + version.push_str(" ("); + version.push_str(&inner.short_sha); + version.push(' '); + version.push_str(&inner.commit_date); + version.push(')'); + } + version + } + + /// Returns whether this directory has a `.git` directory which should be managed by bootstrap. + pub fn is_managed_git_subrepository(&self) -> bool { + match self { + GitInfo::Absent | GitInfo::RecordedForTarball(_) => false, + GitInfo::Present(_) => true, + } + } + + /// Returns whether this is being built from a tarball. + pub fn is_from_tarball(&self) -> bool { + match self { + GitInfo::Absent | GitInfo::Present(_) => false, + GitInfo::RecordedForTarball(_) => true, + } + } +} + +/// Read the commit information from the `git-commit-info` file given the +/// project root. +pub fn read_commit_info_file(root: &Path) -> Option { + if let Ok(contents) = fs::read_to_string(root.join("git-commit-info")) { + let mut lines = contents.lines(); + let sha = lines.next(); + let short_sha = lines.next(); + let commit_date = lines.next(); + let info = match (commit_date, sha, short_sha) { + (Some(commit_date), Some(sha), Some(short_sha)) => Info { + commit_date: commit_date.to_owned(), + sha: sha.to_owned(), + short_sha: short_sha.to_owned(), + }, + _ => panic!("the `git-commit-info` file is malformed"), + }; + Some(info) + } else { + None + } +} + +/// Write the commit information to the `git-commit-info` file given the project +/// root. +pub fn write_commit_info_file(root: &Path, info: &Info) { + let commit_info = format!("{}\n{}\n{}\n", info.sha, info.short_sha, info.commit_date); + t!(fs::write(root.join("git-commit-info"), &commit_info)); +} + +/// Write the commit hash to the `git-commit-hash` file given the project root. +pub fn write_commit_hash_file(root: &Path, sha: &str) { + t!(fs::write(root.join("git-commit-hash"), sha)); +} diff --git a/src/bootstrap/src/utils/dylib.rs b/src/bootstrap/src/utils/dylib.rs new file mode 100644 index 000000000..279a6a010 --- /dev/null +++ b/src/bootstrap/src/utils/dylib.rs @@ -0,0 +1,27 @@ +//! Various utilities for working with dylib paths. + +/// Returns the environment variable which the dynamic library lookup path +/// resides in for this platform. +pub fn dylib_path_var() -> &'static str { + if cfg!(target_os = "windows") { + "PATH" + } else if cfg!(target_os = "macos") { + "DYLD_LIBRARY_PATH" + } else if cfg!(target_os = "haiku") { + "LIBRARY_PATH" + } else if cfg!(target_os = "aix") { + "LIBPATH" + } else { + "LD_LIBRARY_PATH" + } +} + +/// Parses the `dylib_path_var()` environment variable, returning a list of +/// paths that are members of this lookup path. +pub fn dylib_path() -> Vec { + let var = match std::env::var_os(dylib_path_var()) { + Some(v) => v, + None => return vec![], + }; + std::env::split_paths(&var).collect() +} diff --git a/src/bootstrap/src/utils/exec.rs b/src/bootstrap/src/utils/exec.rs new file mode 100644 index 000000000..0aede2022 --- /dev/null +++ b/src/bootstrap/src/utils/exec.rs @@ -0,0 +1,60 @@ +use std::process::Command; + +/// What should be done when the command fails. +#[derive(Debug, Copy, Clone)] +pub enum BehaviorOnFailure { + /// Immediately stop bootstrap. + Exit, + /// Delay failure until the end of bootstrap invocation. + DelayFail, + /// Ignore the failure, the command can fail in an expected way. + Ignore, +} + +/// How should the output of the command be handled. +#[derive(Debug, Copy, Clone)] +pub enum OutputMode { + /// Print both the output (by inheriting stdout/stderr) and also the command itself, if it + /// fails. + PrintAll, + /// Print the output (by inheriting stdout/stderr). + PrintOutput, + /// Suppress the output if the command succeeds, otherwise print the output. + SuppressOnSuccess, +} + +/// Wrapper around `std::process::Command`. +#[derive(Debug)] +pub struct BootstrapCommand<'a> { + pub command: &'a mut Command, + pub failure_behavior: BehaviorOnFailure, + pub output_mode: OutputMode, +} + +impl<'a> BootstrapCommand<'a> { + pub fn delay_failure(self) -> Self { + Self { failure_behavior: BehaviorOnFailure::DelayFail, ..self } + } + + pub fn fail_fast(self) -> Self { + Self { failure_behavior: BehaviorOnFailure::Exit, ..self } + } + + pub fn allow_failure(self) -> Self { + Self { failure_behavior: BehaviorOnFailure::Ignore, ..self } + } + + pub fn output_mode(self, output_mode: OutputMode) -> Self { + Self { output_mode, ..self } + } +} + +impl<'a> From<&'a mut Command> for BootstrapCommand<'a> { + fn from(command: &'a mut Command) -> Self { + Self { + command, + failure_behavior: BehaviorOnFailure::Exit, + output_mode: OutputMode::PrintAll, + } + } +} diff --git a/src/bootstrap/src/utils/helpers.rs b/src/bootstrap/src/utils/helpers.rs new file mode 100644 index 000000000..5bc81f2d9 --- /dev/null +++ b/src/bootstrap/src/utils/helpers.rs @@ -0,0 +1,472 @@ +//! Various utility functions used throughout rustbuild. +//! +//! Simple things like testing the various filesystem operations here and there, +//! not a lot of interesting happenings here unfortunately. + +use build_helper::util::fail; +use std::env; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::str; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +use crate::core::builder::Builder; +use crate::core::config::{Config, TargetSelection}; +use crate::OnceCell; + +pub use crate::utils::dylib::{dylib_path, dylib_path_var}; + +/// A helper macro to `unwrap` a result except also print out details like: +/// +/// * The file/line of the panic +/// * The expression that failed +/// * The error itself +/// +/// This is currently used judiciously throughout the build system rather than +/// using a `Result` with `try!`, but this may change one day... +#[macro_export] +macro_rules! t { + ($e:expr) => { + match $e { + Ok(e) => e, + Err(e) => panic!("{} failed with {}", stringify!($e), e), + } + }; + // it can show extra info in the second parameter + ($e:expr, $extra:expr) => { + match $e { + Ok(e) => e, + Err(e) => panic!("{} failed with {} ({:?})", stringify!($e), e, $extra), + } + }; +} +pub use t; + +/// Given an executable called `name`, return the filename for the +/// executable for a particular target. +pub fn exe(name: &str, target: TargetSelection) -> String { + if target.contains("windows") { + format!("{name}.exe") + } else if target.contains("uefi") { + format!("{name}.efi") + } else { + name.to_string() + } +} + +/// Returns `true` if the file name given looks like a dynamic library. +pub fn is_dylib(name: &str) -> bool { + name.ends_with(".dylib") || name.ends_with(".so") || name.ends_with(".dll") +} + +/// Returns `true` if the file name given looks like a debug info file +pub fn is_debug_info(name: &str) -> bool { + // FIXME: consider split debug info on other platforms (e.g., Linux, macOS) + name.ends_with(".pdb") +} + +/// Returns the corresponding relative library directory that the compiler's +/// dylibs will be found in. +pub fn libdir(target: TargetSelection) -> &'static str { + if target.contains("windows") { "bin" } else { "lib" } +} + +/// Adds a list of lookup paths to `cmd`'s dynamic library lookup path. +/// If the dylib_path_var is already set for this cmd, the old value will be overwritten! +pub fn add_dylib_path(path: Vec, cmd: &mut Command) { + let mut list = dylib_path(); + for path in path { + list.insert(0, path); + } + cmd.env(dylib_path_var(), t!(env::join_paths(list))); +} + +/// Adds a list of lookup paths to `cmd`'s link library lookup path. +pub fn add_link_lib_path(path: Vec, cmd: &mut Command) { + let mut list = link_lib_path(); + for path in path { + list.insert(0, path); + } + cmd.env(link_lib_path_var(), t!(env::join_paths(list))); +} + +/// Returns the environment variable which the link library lookup path +/// resides in for this platform. +fn link_lib_path_var() -> &'static str { + if cfg!(target_env = "msvc") { "LIB" } else { "LIBRARY_PATH" } +} + +/// Parses the `link_lib_path_var()` environment variable, returning a list of +/// paths that are members of this lookup path. +fn link_lib_path() -> Vec { + let var = match env::var_os(link_lib_path_var()) { + Some(v) => v, + None => return vec![], + }; + env::split_paths(&var).collect() +} + +pub struct TimeIt(bool, Instant); + +/// Returns an RAII structure that prints out how long it took to drop. +pub fn timeit(builder: &Builder<'_>) -> TimeIt { + TimeIt(builder.config.dry_run(), Instant::now()) +} + +impl Drop for TimeIt { + fn drop(&mut self) { + let time = self.1.elapsed(); + if !self.0 { + println!("\tfinished in {}.{:03} seconds", time.as_secs(), time.subsec_millis()); + } + } +} + +/// Used for download caching +pub(crate) fn program_out_of_date(stamp: &Path, key: &str) -> bool { + if !stamp.exists() { + return true; + } + t!(fs::read_to_string(stamp)) != key +} + +/// Symlinks two directories, using junctions on Windows and normal symlinks on +/// Unix. +pub fn symlink_dir(config: &Config, original: &Path, link: &Path) -> io::Result<()> { + if config.dry_run() { + return Ok(()); + } + let _ = fs::remove_dir(link); + return symlink_dir_inner(original, link); + + #[cfg(not(windows))] + fn symlink_dir_inner(original: &Path, link: &Path) -> io::Result<()> { + use std::os::unix::fs; + fs::symlink(original, link) + } + + #[cfg(windows)] + fn symlink_dir_inner(target: &Path, junction: &Path) -> io::Result<()> { + junction::create(&target, &junction) + } +} + +pub fn forcing_clang_based_tests() -> bool { + if let Some(var) = env::var_os("RUSTBUILD_FORCE_CLANG_BASED_TESTS") { + match &var.to_string_lossy().to_lowercase()[..] { + "1" | "yes" | "on" => true, + "0" | "no" | "off" => false, + other => { + // Let's make sure typos don't go unnoticed + panic!( + "Unrecognized option '{other}' set in \ + RUSTBUILD_FORCE_CLANG_BASED_TESTS" + ) + } + } + } else { + false + } +} + +pub fn use_host_linker(target: TargetSelection) -> bool { + // FIXME: this information should be gotten by checking the linker flavor + // of the rustc target + !(target.contains("emscripten") + || target.contains("wasm32") + || target.contains("nvptx") + || target.contains("fortanix") + || target.contains("fuchsia") + || target.contains("bpf") + || target.contains("switch")) +} + +pub fn target_supports_cranelift_backend(target: TargetSelection) -> bool { + if target.contains("linux") { + target.contains("x86_64") + || target.contains("aarch64") + || target.contains("s390x") + || target.contains("riscv64gc") + } else if target.contains("darwin") || target.contains("windows") { + target.contains("x86_64") + } else { + false + } +} + +pub fn is_valid_test_suite_arg<'a, P: AsRef>( + path: &'a Path, + suite_path: P, + builder: &Builder<'_>, +) -> Option<&'a str> { + let suite_path = suite_path.as_ref(); + let path = match path.strip_prefix(".") { + Ok(p) => p, + Err(_) => path, + }; + if !path.starts_with(suite_path) { + return None; + } + let abs_path = builder.src.join(path); + let exists = abs_path.is_dir() || abs_path.is_file(); + if !exists { + panic!( + "Invalid test suite filter \"{}\": file or directory does not exist", + abs_path.display() + ); + } + // Since test suite paths are themselves directories, if we don't + // specify a directory or file, we'll get an empty string here + // (the result of the test suite directory without its suite prefix). + // Therefore, we need to filter these out, as only the first --test-args + // flag is respected, so providing an empty --test-args conflicts with + // any following it. + match path.strip_prefix(suite_path).ok().and_then(|p| p.to_str()) { + Some(s) if !s.is_empty() => Some(s), + _ => None, + } +} + +pub fn check_run(cmd: &mut Command, print_cmd_on_fail: bool) -> bool { + let status = match cmd.status() { + Ok(status) => status, + Err(e) => { + println!("failed to execute command: {cmd:?}\nERROR: {e}"); + return false; + } + }; + if !status.success() && print_cmd_on_fail { + println!( + "\n\ncommand did not execute successfully: {cmd:?}\n\ + expected success, got: {status}\n\n" + ); + } + status.success() +} + +pub fn make(host: &str) -> PathBuf { + if host.contains("dragonfly") + || host.contains("freebsd") + || host.contains("netbsd") + || host.contains("openbsd") + { + PathBuf::from("gmake") + } else { + PathBuf::from("make") + } +} + +#[track_caller] +pub fn output(cmd: &mut Command) -> String { + let output = match cmd.stderr(Stdio::inherit()).output() { + Ok(status) => status, + Err(e) => fail(&format!("failed to execute command: {cmd:?}\nERROR: {e}")), + }; + if !output.status.success() { + panic!( + "command did not execute successfully: {:?}\n\ + expected success, got: {}", + cmd, output.status + ); + } + String::from_utf8(output.stdout).unwrap() +} + +/// Returns the last-modified time for `path`, or zero if it doesn't exist. +pub fn mtime(path: &Path) -> SystemTime { + fs::metadata(path).and_then(|f| f.modified()).unwrap_or(UNIX_EPOCH) +} + +/// Returns `true` if `dst` is up to date given that the file or files in `src` +/// are used to generate it. +/// +/// Uses last-modified time checks to verify this. +pub fn up_to_date(src: &Path, dst: &Path) -> bool { + if !dst.exists() { + return false; + } + let threshold = mtime(dst); + let meta = match fs::metadata(src) { + Ok(meta) => meta, + Err(e) => panic!("source {src:?} failed to get metadata: {e}"), + }; + if meta.is_dir() { + dir_up_to_date(src, threshold) + } else { + meta.modified().unwrap_or(UNIX_EPOCH) <= threshold + } +} + +fn dir_up_to_date(src: &Path, threshold: SystemTime) -> bool { + t!(fs::read_dir(src)).map(|e| t!(e)).all(|e| { + let meta = t!(e.metadata()); + if meta.is_dir() { + dir_up_to_date(&e.path(), threshold) + } else { + meta.modified().unwrap_or(UNIX_EPOCH) < threshold + } + }) +} + +/// Copied from `std::path::absolute` until it stabilizes. +/// +/// FIXME: this shouldn't exist. +pub(crate) fn absolute(path: &Path) -> PathBuf { + if path.as_os_str().is_empty() { + panic!("can't make empty path absolute"); + } + #[cfg(unix)] + { + t!(absolute_unix(path), format!("could not make path absolute: {}", path.display())) + } + #[cfg(windows)] + { + t!(absolute_windows(path), format!("could not make path absolute: {}", path.display())) + } + #[cfg(not(any(unix, windows)))] + { + println!("WARNING: bootstrap is not supported on non-unix platforms"); + t!(std::fs::canonicalize(t!(std::env::current_dir()))).join(path) + } +} + +#[cfg(unix)] +/// Make a POSIX path absolute without changing its semantics. +fn absolute_unix(path: &Path) -> io::Result { + // This is mostly a wrapper around collecting `Path::components`, with + // exceptions made where this conflicts with the POSIX specification. + // See 4.13 Pathname Resolution, IEEE Std 1003.1-2017 + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 + + use std::os::unix::prelude::OsStrExt; + let mut components = path.components(); + let path_os = path.as_os_str().as_bytes(); + + let mut normalized = if path.is_absolute() { + // "If a pathname begins with two successive characters, the + // first component following the leading characters may be + // interpreted in an implementation-defined manner, although more than + // two leading characters shall be treated as a single + // character." + if path_os.starts_with(b"//") && !path_os.starts_with(b"///") { + components.next(); + PathBuf::from("//") + } else { + PathBuf::new() + } + } else { + env::current_dir()? + }; + normalized.extend(components); + + // "Interfaces using pathname resolution may specify additional constraints + // when a pathname that does not name an existing directory contains at + // least one non- character and contains one or more trailing + // characters". + // A trailing is also meaningful if "a symbolic link is + // encountered during pathname resolution". + + if path_os.ends_with(b"/") { + normalized.push(""); + } + + Ok(normalized) +} + +#[cfg(windows)] +fn absolute_windows(path: &std::path::Path) -> std::io::Result { + use std::ffi::OsString; + use std::io::Error; + use std::os::windows::ffi::{OsStrExt, OsStringExt}; + use std::ptr::null_mut; + #[link(name = "kernel32")] + extern "system" { + fn GetFullPathNameW( + lpFileName: *const u16, + nBufferLength: u32, + lpBuffer: *mut u16, + lpFilePart: *mut *const u16, + ) -> u32; + } + + unsafe { + // encode the path as UTF-16 + let path: Vec = path.as_os_str().encode_wide().chain([0]).collect(); + let mut buffer = Vec::new(); + // Loop until either success or failure. + loop { + // Try to get the absolute path + let len = GetFullPathNameW( + path.as_ptr(), + buffer.len().try_into().unwrap(), + buffer.as_mut_ptr(), + null_mut(), + ); + match len as usize { + // Failure + 0 => return Err(Error::last_os_error()), + // Buffer is too small, resize. + len if len > buffer.len() => buffer.resize(len, 0), + // Success! + len => { + buffer.truncate(len); + return Ok(OsString::from_wide(&buffer).into()); + } + } + } + } +} + +/// Adapted from +/// +/// When `clang-cl` is used with instrumentation, we need to add clang's runtime library resource +/// directory to the linker flags, otherwise there will be linker errors about the profiler runtime +/// missing. This function returns the path to that directory. +pub fn get_clang_cl_resource_dir(clang_cl_path: &str) -> PathBuf { + // Similar to how LLVM does it, to find clang's library runtime directory: + // - we ask `clang-cl` to locate the `clang_rt.builtins` lib. + let mut builtins_locator = Command::new(clang_cl_path); + builtins_locator.args(&["/clang:-print-libgcc-file-name", "/clang:--rtlib=compiler-rt"]); + + let clang_rt_builtins = output(&mut builtins_locator); + let clang_rt_builtins = Path::new(clang_rt_builtins.trim()); + assert!( + clang_rt_builtins.exists(), + "`clang-cl` must correctly locate the library runtime directory" + ); + + // - the profiler runtime will be located in the same directory as the builtins lib, like + // `$LLVM_DISTRO_ROOT/lib/clang/$LLVM_VERSION/lib/windows`. + let clang_rt_dir = clang_rt_builtins.parent().expect("The clang lib folder should exist"); + clang_rt_dir.to_path_buf() +} + +pub fn lld_flag_no_threads(is_windows: bool) -> &'static str { + static LLD_NO_THREADS: OnceCell<(&'static str, &'static str)> = OnceCell::new(); + let (windows, other) = LLD_NO_THREADS.get_or_init(|| { + let out = output(Command::new("lld").arg("-flavor").arg("ld").arg("--version")); + let newer = match (out.find(char::is_numeric), out.find('.')) { + (Some(b), Some(e)) => out.as_str()[b..e].parse::().ok().unwrap_or(14) > 10, + _ => true, + }; + if newer { ("/threads:1", "--threads=1") } else { ("/no-threads", "--no-threads") } + }); + if is_windows { windows } else { other } +} + +pub fn dir_is_empty(dir: &Path) -> bool { + t!(std::fs::read_dir(dir)).next().is_none() +} + +/// Extract the beta revision from the full version string. +/// +/// The full version string looks like "a.b.c-beta.y". And we need to extract +/// the "y" part from the string. +pub fn extract_beta_rev(version: &str) -> Option { + let parts = version.splitn(2, "-beta.").collect::>(); + let count = parts.get(1).and_then(|s| s.find(' ').map(|p| (&s[..p]).to_string())); + + count +} diff --git a/src/bootstrap/src/utils/job.rs b/src/bootstrap/src/utils/job.rs new file mode 100644 index 000000000..c5c718a89 --- /dev/null +++ b/src/bootstrap/src/utils/job.rs @@ -0,0 +1,159 @@ +#[cfg(windows)] +pub use for_windows::*; + +#[cfg(any(target_os = "haiku", target_os = "hermit", not(any(unix, windows))))] +pub unsafe fn setup(_build: &mut crate::Build) {} + +#[cfg(all(unix, not(target_os = "haiku")))] +pub unsafe fn setup(build: &mut crate::Build) { + if build.config.low_priority { + libc::setpriority(libc::PRIO_PGRP as _, 0, 10); + } +} + +#[cfg(windows)] +mod for_windows { + //! Job management on Windows for bootstrapping + //! + //! Most of the time when you're running a build system (e.g., make) you expect + //! Ctrl-C or abnormal termination to actually terminate the entire tree of + //! process in play, not just the one at the top. This currently works "by + //! default" on Unix platforms because Ctrl-C actually sends a signal to the + //! *process group* rather than the parent process, so everything will get torn + //! down. On Windows, however, this does not happen and Ctrl-C just kills the + //! parent process. + //! + //! To achieve the same semantics on Windows we use Job Objects to ensure that + //! all processes die at the same time. Job objects have a mode of operation + //! where when all handles to the object are closed it causes all child + //! processes associated with the object to be terminated immediately. + //! Conveniently whenever a process in the job object spawns a new process the + //! child will be associated with the job object as well. This means if we add + //! ourselves to the job object we create then everything will get torn down! + //! + //! Unfortunately most of the time the build system is actually called from a + //! python wrapper (which manages things like building the build system) so this + //! all doesn't quite cut it so far. To go the last mile we duplicate the job + //! object handle into our parent process (a python process probably) and then + //! close our own handle. This means that the only handle to the job object + //! resides in the parent python process, so when python dies the whole build + //! system dies (as one would probably expect!). + //! + //! Note that this module has a #[cfg(windows)] above it as none of this logic + //! is required on Unix. + + use crate::Build; + use std::env; + use std::ffi::c_void; + use std::io; + use std::mem; + + use windows::{ + core::PCWSTR, + Win32::Foundation::{CloseHandle, DuplicateHandle, DUPLICATE_SAME_ACCESS, HANDLE}, + Win32::System::Diagnostics::Debug::{ + SetErrorMode, SEM_NOGPFAULTERRORBOX, THREAD_ERROR_MODE, + }, + Win32::System::JobObjects::{ + AssignProcessToJobObject, CreateJobObjectW, JobObjectExtendedLimitInformation, + SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, + JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOB_OBJECT_LIMIT_PRIORITY_CLASS, + }, + Win32::System::Threading::{ + GetCurrentProcess, OpenProcess, BELOW_NORMAL_PRIORITY_CLASS, PROCESS_DUP_HANDLE, + }, + }; + + pub unsafe fn setup(build: &mut Build) { + // Enable the Windows Error Reporting dialog which msys disables, + // so we can JIT debug rustc + let mode = SetErrorMode(THREAD_ERROR_MODE::default()); + let mode = THREAD_ERROR_MODE(mode); + SetErrorMode(mode & !SEM_NOGPFAULTERRORBOX); + + // Create a new job object for us to use + let job = CreateJobObjectW(None, PCWSTR::null()).unwrap(); + + // Indicate that when all handles to the job object are gone that all + // process in the object should be killed. Note that this includes our + // entire process tree by default because we've added ourselves and our + // children will reside in the job by default. + let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default(); + info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + if build.config.low_priority { + info.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_PRIORITY_CLASS; + info.BasicLimitInformation.PriorityClass = BELOW_NORMAL_PRIORITY_CLASS.0; + } + let r = SetInformationJobObject( + job, + JobObjectExtendedLimitInformation, + &info as *const _ as *const c_void, + mem::size_of_val(&info) as u32, + ); + assert!(r.is_ok(), "{}", io::Error::last_os_error()); + + // Assign our process to this job object. Note that if this fails, one very + // likely reason is that we are ourselves already in a job object! This can + // happen on the build bots that we've got for Windows, or if just anyone + // else is instrumenting the build. In this case we just bail out + // immediately and assume that they take care of it. + // + // Also note that nested jobs (why this might fail) are supported in recent + // versions of Windows, but the version of Windows that our bots are running + // at least don't support nested job objects. + let r = AssignProcessToJobObject(job, GetCurrentProcess()); + if r.is_err() { + CloseHandle(job).ok(); + return; + } + + // If we've got a parent process (e.g., the python script that called us) + // then move ownership of this job object up to them. That way if the python + // script is killed (e.g., via ctrl-c) then we'll all be torn down. + // + // If we don't have a parent (e.g., this was run directly) then we + // intentionally leak the job object handle. When our process exits + // (normally or abnormally) it will close the handle implicitly, causing all + // processes in the job to be cleaned up. + let pid = match env::var("BOOTSTRAP_PARENT_ID") { + Ok(s) => s, + Err(..) => return, + }; + + let parent = match OpenProcess(PROCESS_DUP_HANDLE, false, pid.parse().unwrap()).ok() { + Some(parent) => parent, + _ => { + // If we get a null parent pointer here, it is possible that either + // we have an invalid pid or the parent process has been closed. + // Since the first case rarely happens + // (only when wrongly setting the environmental variable), + // it might be better to improve the experience of the second case + // when users have interrupted the parent process and we haven't finish + // duplicating the handle yet. We just need close the job object if that occurs. + CloseHandle(job).ok(); + return; + } + }; + + let mut parent_handle = HANDLE::default(); + let r = DuplicateHandle( + GetCurrentProcess(), + job, + parent, + &mut parent_handle, + 0, + false, + DUPLICATE_SAME_ACCESS, + ); + + // If this failed, well at least we tried! An example of DuplicateHandle + // failing in the past has been when the wrong python2 package spawned this + // build system (e.g., the `python2` package in MSYS instead of + // `mingw-w64-x86_64-python2`). Not sure why it failed, but the "failure + // mode" here is that we only clean everything up when the build system + // dies, not when the python parent does, so not too bad. + if r.is_err() { + CloseHandle(job).ok(); + } + } +} diff --git a/src/bootstrap/src/utils/metrics.rs b/src/bootstrap/src/utils/metrics.rs new file mode 100644 index 000000000..174f37422 --- /dev/null +++ b/src/bootstrap/src/utils/metrics.rs @@ -0,0 +1,258 @@ +//! This module is responsible for collecting metrics profiling information for the current build +//! and dumping it to disk as JSON, to aid investigations on build and CI performance. +//! +//! As this module requires additional dependencies not present during local builds, it's cfg'd +//! away whenever the `build.metrics` config option is not set to `true`. + +use crate::core::builder::{Builder, Step}; +use crate::utils::helpers::t; +use crate::Build; +use build_helper::metrics::{ + JsonInvocation, JsonInvocationSystemStats, JsonNode, JsonRoot, JsonStepSystemStats, Test, + TestOutcome, TestSuite, TestSuiteMetadata, +}; +use std::cell::RefCell; +use std::fs::File; +use std::io::BufWriter; +use std::time::{Duration, Instant, SystemTime}; +use sysinfo::{CpuExt, System, SystemExt}; + +// Update this number whenever a breaking change is made to the build metrics. +// +// The output format is versioned for two reasons: +// +// - The metadata is intended to be consumed by external tooling, and exposing a format version +// helps the tools determine whether they're compatible with a metrics file. +// +// - If a developer enables build metrics in their local checkout, making a breaking change to the +// metrics format would result in a hard-to-diagnose error message when an existing metrics file +// is not compatible with the new changes. With a format version number, bootstrap can discard +// incompatible metrics files instead of appending metrics to them. +// +// Version changelog: +// +// - v0: initial version +// - v1: replaced JsonNode::Test with JsonNode::TestSuite +// +const CURRENT_FORMAT_VERSION: usize = 1; + +pub(crate) struct BuildMetrics { + state: RefCell, +} + +/// NOTE: this isn't really cloning anything, but `x suggest` doesn't need metrics so this is probably ok. +impl Clone for BuildMetrics { + fn clone(&self) -> Self { + Self::init() + } +} + +impl BuildMetrics { + pub(crate) fn init() -> Self { + let state = RefCell::new(MetricsState { + finished_steps: Vec::new(), + running_steps: Vec::new(), + + system_info: System::new(), + timer_start: None, + invocation_timer_start: Instant::now(), + invocation_start: SystemTime::now(), + }); + + BuildMetrics { state } + } + + pub(crate) fn enter_step(&self, step: &S, builder: &Builder<'_>) { + // Do not record dry runs, as they'd be duplicates of the actual steps. + if builder.config.dry_run() { + return; + } + + let mut state = self.state.borrow_mut(); + + // Consider all the stats gathered so far as the parent's. + if !state.running_steps.is_empty() { + self.collect_stats(&mut *state); + } + + state.system_info.refresh_cpu(); + state.timer_start = Some(Instant::now()); + + state.running_steps.push(StepMetrics { + type_: std::any::type_name::().into(), + debug_repr: format!("{step:?}"), + + cpu_usage_time_sec: 0.0, + duration_excluding_children_sec: Duration::ZERO, + + children: Vec::new(), + test_suites: Vec::new(), + }); + } + + pub(crate) fn exit_step(&self, builder: &Builder<'_>) { + // Do not record dry runs, as they'd be duplicates of the actual steps. + if builder.config.dry_run() { + return; + } + + let mut state = self.state.borrow_mut(); + + self.collect_stats(&mut *state); + + let step = state.running_steps.pop().unwrap(); + if state.running_steps.is_empty() { + state.finished_steps.push(step); + state.timer_start = None; + } else { + state.running_steps.last_mut().unwrap().children.push(step); + + // Start collecting again for the parent step. + state.system_info.refresh_cpu(); + state.timer_start = Some(Instant::now()); + } + } + + pub(crate) fn begin_test_suite(&self, metadata: TestSuiteMetadata, builder: &Builder<'_>) { + // Do not record dry runs, as they'd be duplicates of the actual steps. + if builder.config.dry_run() { + return; + } + + let mut state = self.state.borrow_mut(); + let step = state.running_steps.last_mut().unwrap(); + step.test_suites.push(TestSuite { metadata, tests: Vec::new() }); + } + + pub(crate) fn record_test(&self, name: &str, outcome: TestOutcome, builder: &Builder<'_>) { + // Do not record dry runs, as they'd be duplicates of the actual steps. + if builder.config.dry_run() { + return; + } + + let mut state = self.state.borrow_mut(); + let step = state.running_steps.last_mut().unwrap(); + + if let Some(test_suite) = step.test_suites.last_mut() { + test_suite.tests.push(Test { name: name.to_string(), outcome }); + } else { + panic!("metrics.record_test() called without calling metrics.begin_test_suite() first"); + } + } + + fn collect_stats(&self, state: &mut MetricsState) { + let step = state.running_steps.last_mut().unwrap(); + + let elapsed = state.timer_start.unwrap().elapsed(); + step.duration_excluding_children_sec += elapsed; + + state.system_info.refresh_cpu(); + let cpu = state.system_info.cpus().iter().map(|p| p.cpu_usage()).sum::(); + step.cpu_usage_time_sec += cpu as f64 / 100.0 * elapsed.as_secs_f64(); + } + + pub(crate) fn persist(&self, build: &Build) { + let mut state = self.state.borrow_mut(); + assert!(state.running_steps.is_empty(), "steps are still executing"); + + let dest = build.out.join("metrics.json"); + + let mut system = System::new(); + system.refresh_cpu(); + system.refresh_memory(); + + let system_stats = JsonInvocationSystemStats { + cpu_threads_count: system.cpus().len(), + cpu_model: system.cpus()[0].brand().into(), + + memory_total_bytes: system.total_memory(), + }; + let steps = std::mem::take(&mut state.finished_steps); + + // Some of our CI builds consist of multiple independent CI invocations. Ensure all the + // previous invocations are still present in the resulting file. + let mut invocations = match std::fs::read(&dest) { + Ok(contents) => { + // We first parse just the format_version field to have the check succeed even if + // the rest of the contents are not valid anymore. + let version: OnlyFormatVersion = t!(serde_json::from_slice(&contents)); + if version.format_version == CURRENT_FORMAT_VERSION { + t!(serde_json::from_slice::(&contents)).invocations + } else { + println!( + "WARNING: overriding existing build/metrics.json, as it's not \ + compatible with build metrics format version {CURRENT_FORMAT_VERSION}." + ); + Vec::new() + } + } + Err(err) => { + if err.kind() != std::io::ErrorKind::NotFound { + panic!("failed to open existing metrics file at {}: {err}", dest.display()); + } + Vec::new() + } + }; + invocations.push(JsonInvocation { + start_time: state + .invocation_start + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(), + duration_including_children_sec: state.invocation_timer_start.elapsed().as_secs_f64(), + children: steps.into_iter().map(|step| self.prepare_json_step(step)).collect(), + }); + + let json = JsonRoot { format_version: CURRENT_FORMAT_VERSION, system_stats, invocations }; + + t!(std::fs::create_dir_all(dest.parent().unwrap())); + let mut file = BufWriter::new(t!(File::create(&dest))); + t!(serde_json::to_writer(&mut file, &json)); + } + + fn prepare_json_step(&self, step: StepMetrics) -> JsonNode { + let mut children = Vec::new(); + children.extend(step.children.into_iter().map(|child| self.prepare_json_step(child))); + children.extend(step.test_suites.into_iter().map(JsonNode::TestSuite)); + + JsonNode::RustbuildStep { + type_: step.type_, + debug_repr: step.debug_repr, + + duration_excluding_children_sec: step.duration_excluding_children_sec.as_secs_f64(), + system_stats: JsonStepSystemStats { + cpu_utilization_percent: step.cpu_usage_time_sec * 100.0 + / step.duration_excluding_children_sec.as_secs_f64(), + }, + + children, + } + } +} + +struct MetricsState { + finished_steps: Vec, + running_steps: Vec, + + system_info: System, + timer_start: Option, + invocation_timer_start: Instant, + invocation_start: SystemTime, +} + +struct StepMetrics { + type_: String, + debug_repr: String, + + cpu_usage_time_sec: f64, + duration_excluding_children_sec: Duration, + + children: Vec, + test_suites: Vec, +} + +#[derive(serde_derive::Deserialize)] +struct OnlyFormatVersion { + #[serde(default)] // For version 0 the field was not present. + format_version: usize, +} diff --git a/src/bootstrap/src/utils/mod.rs b/src/bootstrap/src/utils/mod.rs new file mode 100644 index 000000000..8ca22d008 --- /dev/null +++ b/src/bootstrap/src/utils/mod.rs @@ -0,0 +1,15 @@ +//! This module contains integral components of the build and configuration process, providing +//! support for a wide range of tasks and operations such as caching, tarballs, release +//! channels, job management, etc. + +pub(crate) mod cache; +pub(crate) mod cc_detect; +pub(crate) mod channel; +pub(crate) mod dylib; +pub(crate) mod exec; +pub(crate) mod helpers; +pub(crate) mod job; +#[cfg(feature = "build-metrics")] +pub(crate) mod metrics; +pub(crate) mod render_tests; +pub(crate) mod tarball; diff --git a/src/bootstrap/src/utils/render_tests.rs b/src/bootstrap/src/utils/render_tests.rs new file mode 100644 index 000000000..bff47f65c --- /dev/null +++ b/src/bootstrap/src/utils/render_tests.rs @@ -0,0 +1,400 @@ +//! This module renders the JSON output of libtest into a human-readable form, trying to be as +//! similar to libtest's native output as possible. +//! +//! This is needed because we need to use libtest in JSON mode to extract granular information +//! about the executed tests. Doing so suppresses the human-readable output, and (compared to Cargo +//! and rustc) libtest doesn't include the rendered human-readable output as a JSON field. We had +//! to reimplement all the rendering logic in this module because of that. + +use crate::core::builder::Builder; +use std::io::{BufRead, BufReader, Read, Write}; +use std::process::{ChildStdout, Command, Stdio}; +use std::time::Duration; +use termcolor::{Color, ColorSpec, WriteColor}; + +const TERSE_TESTS_PER_LINE: usize = 88; + +pub(crate) fn add_flags_and_try_run_tests(builder: &Builder<'_>, cmd: &mut Command) -> bool { + if cmd.get_args().position(|arg| arg == "--").is_none() { + cmd.arg("--"); + } + cmd.args(&["-Z", "unstable-options", "--format", "json"]); + + try_run_tests(builder, cmd, false) +} + +pub(crate) fn try_run_tests(builder: &Builder<'_>, cmd: &mut Command, stream: bool) -> bool { + if builder.config.dry_run() { + return true; + } + + if !run_tests(builder, cmd, stream) { + if builder.fail_fast { + crate::exit!(1); + } else { + let mut failures = builder.delayed_failures.borrow_mut(); + failures.push(format!("{cmd:?}")); + false + } + } else { + true + } +} + +fn run_tests(builder: &Builder<'_>, cmd: &mut Command, stream: bool) -> bool { + cmd.stdout(Stdio::piped()); + + builder.verbose(&format!("running: {cmd:?}")); + + let mut process = cmd.spawn().unwrap(); + + // This runs until the stdout of the child is closed, which means the child exited. We don't + // run this on another thread since the builder is not Sync. + let renderer = Renderer::new(process.stdout.take().unwrap(), builder); + if stream { + renderer.stream_all(); + } else { + renderer.render_all(); + } + + let result = process.wait_with_output().unwrap(); + if !result.status.success() && builder.is_verbose() { + println!( + "\n\ncommand did not execute successfully: {cmd:?}\n\ + expected success, got: {}", + result.status + ); + } + + result.status.success() +} + +struct Renderer<'a> { + stdout: BufReader, + failures: Vec, + benches: Vec, + builder: &'a Builder<'a>, + tests_count: Option, + executed_tests: usize, + terse_tests_in_line: usize, +} + +impl<'a> Renderer<'a> { + fn new(stdout: ChildStdout, builder: &'a Builder<'a>) -> Self { + Self { + stdout: BufReader::new(stdout), + benches: Vec::new(), + failures: Vec::new(), + builder, + tests_count: None, + executed_tests: 0, + terse_tests_in_line: 0, + } + } + + fn render_all(mut self) { + let mut line = Vec::new(); + loop { + line.clear(); + match self.stdout.read_until(b'\n', &mut line) { + Ok(_) => {} + Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => break, + Err(err) => panic!("failed to read output of test runner: {err}"), + } + if line.is_empty() { + break; + } + + match serde_json::from_slice(&line) { + Ok(parsed) => self.render_message(parsed), + Err(_err) => { + // Handle non-JSON output, for example when --nocapture is passed. + let mut stdout = std::io::stdout(); + stdout.write_all(&line).unwrap(); + let _ = stdout.flush(); + } + } + } + } + + /// Renders the stdout characters one by one + fn stream_all(mut self) { + let mut buffer = [0; 1]; + loop { + match self.stdout.read(&mut buffer) { + Ok(0) => break, + Ok(_) => { + let mut stdout = std::io::stdout(); + stdout.write_all(&buffer).unwrap(); + let _ = stdout.flush(); + } + Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => break, + Err(err) => panic!("failed to read output of test runner: {err}"), + } + } + } + + fn render_test_outcome(&mut self, outcome: Outcome<'_>, test: &TestOutcome) { + self.executed_tests += 1; + + #[cfg(feature = "build-metrics")] + self.builder.metrics.record_test( + &test.name, + match outcome { + Outcome::Ok | Outcome::BenchOk => build_helper::metrics::TestOutcome::Passed, + Outcome::Failed => build_helper::metrics::TestOutcome::Failed, + Outcome::Ignored { reason } => build_helper::metrics::TestOutcome::Ignored { + ignore_reason: reason.map(|s| s.to_string()), + }, + }, + self.builder, + ); + + if self.builder.config.verbose_tests { + self.render_test_outcome_verbose(outcome, test); + } else { + self.render_test_outcome_terse(outcome, test); + } + } + + fn render_test_outcome_verbose(&self, outcome: Outcome<'_>, test: &TestOutcome) { + print!("test {} ... ", test.name); + self.builder.colored_stdout(|stdout| outcome.write_long(stdout)).unwrap(); + if let Some(exec_time) = test.exec_time { + print!(" ({exec_time:.2?})"); + } + println!(); + } + + fn render_test_outcome_terse(&mut self, outcome: Outcome<'_>, _: &TestOutcome) { + if self.terse_tests_in_line != 0 && self.terse_tests_in_line % TERSE_TESTS_PER_LINE == 0 { + if let Some(total) = self.tests_count { + let total = total.to_string(); + let executed = format!("{:>width$}", self.executed_tests - 1, width = total.len()); + print!(" {executed}/{total}"); + } + println!(); + self.terse_tests_in_line = 0; + } + + self.terse_tests_in_line += 1; + self.builder.colored_stdout(|stdout| outcome.write_short(stdout)).unwrap(); + let _ = std::io::stdout().flush(); + } + + fn render_suite_outcome(&self, outcome: Outcome<'_>, suite: &SuiteOutcome) { + // The terse output doesn't end with a newline, so we need to add it ourselves. + if !self.builder.config.verbose_tests { + println!(); + } + + if !self.failures.is_empty() { + println!("\nfailures:\n"); + for failure in &self.failures { + if failure.stdout.is_some() || failure.message.is_some() { + println!("---- {} stdout ----", failure.name); + if let Some(stdout) = &failure.stdout { + println!("{stdout}"); + } + if let Some(message) = &failure.message { + println!("NOTE: {message}"); + } + } + } + + println!("\nfailures:"); + for failure in &self.failures { + println!(" {}", failure.name); + } + } + + if !self.benches.is_empty() { + println!("\nbenchmarks:"); + + let mut rows = Vec::new(); + for bench in &self.benches { + rows.push(( + &bench.name, + format!("{:.2?}/iter", Duration::from_nanos(bench.median)), + format!("+/- {:.2?}", Duration::from_nanos(bench.deviation)), + )); + } + + let max_0 = rows.iter().map(|r| r.0.len()).max().unwrap_or(0); + let max_1 = rows.iter().map(|r| r.1.len()).max().unwrap_or(0); + let max_2 = rows.iter().map(|r| r.2.len()).max().unwrap_or(0); + for row in &rows { + println!(" {:max_1$} {:>max_2$}", row.0, row.1, row.2); + } + } + + print!("\ntest result: "); + self.builder.colored_stdout(|stdout| outcome.write_long(stdout)).unwrap(); + println!( + ". {} passed; {} failed; {} ignored; {} measured; {} filtered out; \ + finished in {:.2?}\n", + suite.passed, + suite.failed, + suite.ignored, + suite.measured, + suite.filtered_out, + Duration::from_secs_f64(suite.exec_time) + ); + } + + fn render_message(&mut self, message: Message) { + match message { + Message::Suite(SuiteMessage::Started { test_count }) => { + println!("\nrunning {test_count} tests"); + self.executed_tests = 0; + self.terse_tests_in_line = 0; + self.tests_count = Some(test_count); + } + Message::Suite(SuiteMessage::Ok(outcome)) => { + self.render_suite_outcome(Outcome::Ok, &outcome); + } + Message::Suite(SuiteMessage::Failed(outcome)) => { + self.render_suite_outcome(Outcome::Failed, &outcome); + } + Message::Bench(outcome) => { + // The formatting for benchmarks doesn't replicate 1:1 the formatting libtest + // outputs, mostly because libtest's formatting is broken in terse mode, which is + // the default used by our monorepo. We use a different formatting instead: + // successful benchmarks are just showed as "benchmarked"/"b", and the details are + // outputted at the bottom like failures. + let fake_test_outcome = TestOutcome { + name: outcome.name.clone(), + exec_time: None, + stdout: None, + message: None, + }; + self.render_test_outcome(Outcome::BenchOk, &fake_test_outcome); + self.benches.push(outcome); + } + Message::Test(TestMessage::Ok(outcome)) => { + self.render_test_outcome(Outcome::Ok, &outcome); + } + Message::Test(TestMessage::Ignored(outcome)) => { + self.render_test_outcome( + Outcome::Ignored { reason: outcome.message.as_deref() }, + &outcome, + ); + } + Message::Test(TestMessage::Failed(outcome)) => { + self.render_test_outcome(Outcome::Failed, &outcome); + self.failures.push(outcome); + } + Message::Test(TestMessage::Timeout { name }) => { + println!("test {name} has been running for a long time"); + } + Message::Test(TestMessage::Started) => {} // Not useful + } + } +} + +enum Outcome<'a> { + Ok, + BenchOk, + Failed, + Ignored { reason: Option<&'a str> }, +} + +impl Outcome<'_> { + fn write_short(&self, writer: &mut dyn WriteColor) -> Result<(), std::io::Error> { + match self { + Outcome::Ok => { + writer.set_color(&ColorSpec::new().set_fg(Some(Color::Green)))?; + write!(writer, ".")?; + } + Outcome::BenchOk => { + writer.set_color(&ColorSpec::new().set_fg(Some(Color::Cyan)))?; + write!(writer, "b")?; + } + Outcome::Failed => { + writer.set_color(&ColorSpec::new().set_fg(Some(Color::Red)))?; + write!(writer, "F")?; + } + Outcome::Ignored { .. } => { + writer.set_color(&ColorSpec::new().set_fg(Some(Color::Yellow)))?; + write!(writer, "i")?; + } + } + writer.reset() + } + + fn write_long(&self, writer: &mut dyn WriteColor) -> Result<(), std::io::Error> { + match self { + Outcome::Ok => { + writer.set_color(&ColorSpec::new().set_fg(Some(Color::Green)))?; + write!(writer, "ok")?; + } + Outcome::BenchOk => { + writer.set_color(&ColorSpec::new().set_fg(Some(Color::Cyan)))?; + write!(writer, "benchmarked")?; + } + Outcome::Failed => { + writer.set_color(&ColorSpec::new().set_fg(Some(Color::Red)))?; + write!(writer, "FAILED")?; + } + Outcome::Ignored { reason } => { + writer.set_color(&ColorSpec::new().set_fg(Some(Color::Yellow)))?; + write!(writer, "ignored")?; + if let Some(reason) = reason { + write!(writer, ", {reason}")?; + } + } + } + writer.reset() + } +} + +#[derive(serde_derive::Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +enum Message { + Suite(SuiteMessage), + Test(TestMessage), + Bench(BenchOutcome), +} + +#[derive(serde_derive::Deserialize)] +#[serde(tag = "event", rename_all = "snake_case")] +enum SuiteMessage { + Ok(SuiteOutcome), + Failed(SuiteOutcome), + Started { test_count: usize }, +} + +#[derive(serde_derive::Deserialize)] +struct SuiteOutcome { + passed: usize, + failed: usize, + ignored: usize, + measured: usize, + filtered_out: usize, + exec_time: f64, +} + +#[derive(serde_derive::Deserialize)] +#[serde(tag = "event", rename_all = "snake_case")] +enum TestMessage { + Ok(TestOutcome), + Failed(TestOutcome), + Ignored(TestOutcome), + Timeout { name: String }, + Started, +} + +#[derive(serde_derive::Deserialize)] +struct BenchOutcome { + name: String, + median: u64, + deviation: u64, +} + +#[derive(serde_derive::Deserialize)] +struct TestOutcome { + name: String, + exec_time: Option, + stdout: Option, + message: Option, +} diff --git a/src/bootstrap/src/utils/tarball.rs b/src/bootstrap/src/utils/tarball.rs new file mode 100644 index 000000000..a8393f88f --- /dev/null +++ b/src/bootstrap/src/utils/tarball.rs @@ -0,0 +1,381 @@ +use std::{ + path::{Path, PathBuf}, + process::Command, +}; + +use crate::core::build_steps::dist::distdir; +use crate::core::builder::Builder; +use crate::utils::channel; +use crate::utils::helpers::t; + +#[derive(Copy, Clone)] +pub(crate) enum OverlayKind { + Rust, + LLVM, + Cargo, + Clippy, + Miri, + Rustfmt, + RustDemangler, + RLS, + RustAnalyzer, + RustcCodegenCranelift, +} + +impl OverlayKind { + fn legal_and_readme(&self) -> &[&str] { + match self { + OverlayKind::Rust => &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"], + OverlayKind::LLVM => { + &["src/llvm-project/llvm/LICENSE.TXT", "src/llvm-project/llvm/README.txt"] + } + OverlayKind::Cargo => &[ + "src/tools/cargo/README.md", + "src/tools/cargo/LICENSE-MIT", + "src/tools/cargo/LICENSE-APACHE", + "src/tools/cargo/LICENSE-THIRD-PARTY", + ], + OverlayKind::Clippy => &[ + "src/tools/clippy/README.md", + "src/tools/clippy/LICENSE-APACHE", + "src/tools/clippy/LICENSE-MIT", + ], + OverlayKind::Miri => &[ + "src/tools/miri/README.md", + "src/tools/miri/LICENSE-APACHE", + "src/tools/miri/LICENSE-MIT", + ], + OverlayKind::Rustfmt => &[ + "src/tools/rustfmt/README.md", + "src/tools/rustfmt/LICENSE-APACHE", + "src/tools/rustfmt/LICENSE-MIT", + ], + OverlayKind::RustDemangler => { + &["src/tools/rust-demangler/README.md", "LICENSE-APACHE", "LICENSE-MIT"] + } + OverlayKind::RLS => &["src/tools/rls/README.md", "LICENSE-APACHE", "LICENSE-MIT"], + OverlayKind::RustAnalyzer => &[ + "src/tools/rust-analyzer/README.md", + "src/tools/rust-analyzer/LICENSE-APACHE", + "src/tools/rust-analyzer/LICENSE-MIT", + ], + OverlayKind::RustcCodegenCranelift => &[ + "compiler/rustc_codegen_cranelift/Readme.md", + "compiler/rustc_codegen_cranelift/LICENSE-APACHE", + "compiler/rustc_codegen_cranelift/LICENSE-MIT", + ], + } + } + + fn version(&self, builder: &Builder<'_>) -> String { + match self { + OverlayKind::Rust => builder.rust_version(), + OverlayKind::LLVM => builder.rust_version(), + OverlayKind::RustDemangler => builder.release_num("rust-demangler"), + OverlayKind::Cargo => { + builder.cargo_info.version(builder, &builder.release_num("cargo")) + } + OverlayKind::Clippy => { + builder.clippy_info.version(builder, &builder.release_num("clippy")) + } + OverlayKind::Miri => builder.miri_info.version(builder, &builder.release_num("miri")), + OverlayKind::Rustfmt => { + builder.rustfmt_info.version(builder, &builder.release_num("rustfmt")) + } + OverlayKind::RLS => builder.release(&builder.release_num("rls")), + OverlayKind::RustAnalyzer => builder + .rust_analyzer_info + .version(builder, &builder.release_num("rust-analyzer/crates/rust-analyzer")), + OverlayKind::RustcCodegenCranelift => builder.rust_version(), + } + } +} + +pub(crate) struct Tarball<'a> { + builder: &'a Builder<'a>, + + pkgname: String, + component: String, + target: Option, + product_name: String, + overlay: OverlayKind, + + temp_dir: PathBuf, + image_dir: PathBuf, + overlay_dir: PathBuf, + bulk_dirs: Vec, + + include_target_in_component_name: bool, + is_preview: bool, + permit_symlinks: bool, +} + +impl<'a> Tarball<'a> { + pub(crate) fn new(builder: &'a Builder<'a>, component: &str, target: &str) -> Self { + Self::new_inner(builder, component, Some(target.into())) + } + + pub(crate) fn new_targetless(builder: &'a Builder<'a>, component: &str) -> Self { + Self::new_inner(builder, component, None) + } + + fn new_inner(builder: &'a Builder<'a>, component: &str, target: Option) -> Self { + let pkgname = crate::core::build_steps::dist::pkgname(builder, component); + + let mut temp_dir = builder.out.join("tmp").join("tarball").join(component); + if let Some(target) = &target { + temp_dir = temp_dir.join(target); + } + let _ = std::fs::remove_dir_all(&temp_dir); + + let image_dir = temp_dir.join("image"); + let overlay_dir = temp_dir.join("overlay"); + + Self { + builder, + + pkgname, + component: component.into(), + target, + product_name: "Rust".into(), + overlay: OverlayKind::Rust, + + temp_dir, + image_dir, + overlay_dir, + bulk_dirs: Vec::new(), + + include_target_in_component_name: false, + is_preview: false, + permit_symlinks: false, + } + } + + pub(crate) fn set_overlay(&mut self, overlay: OverlayKind) { + self.overlay = overlay; + } + + pub(crate) fn set_product_name(&mut self, name: &str) { + self.product_name = name.into(); + } + + pub(crate) fn include_target_in_component_name(&mut self, include: bool) { + self.include_target_in_component_name = include; + } + + pub(crate) fn is_preview(&mut self, is: bool) { + self.is_preview = is; + } + + pub(crate) fn permit_symlinks(&mut self, flag: bool) { + self.permit_symlinks = flag; + } + + pub(crate) fn image_dir(&self) -> &Path { + t!(std::fs::create_dir_all(&self.image_dir)); + &self.image_dir + } + + pub(crate) fn add_file(&self, src: impl AsRef, destdir: impl AsRef, perms: u32) { + // create_dir_all fails to create `foo/bar/.`, so when the destination is "." this simply + // uses the base directory as the destination directory. + let destdir = if destdir.as_ref() == Path::new(".") { + self.image_dir.clone() + } else { + self.image_dir.join(destdir.as_ref()) + }; + + t!(std::fs::create_dir_all(&destdir)); + self.builder.install(src.as_ref(), &destdir, perms); + } + + pub(crate) fn add_renamed_file( + &self, + src: impl AsRef, + destdir: impl AsRef, + new_name: &str, + ) { + let destdir = self.image_dir.join(destdir.as_ref()); + t!(std::fs::create_dir_all(&destdir)); + self.builder.copy(src.as_ref(), &destdir.join(new_name)); + } + + pub(crate) fn add_legal_and_readme_to(&self, destdir: impl AsRef) { + for file in self.overlay.legal_and_readme() { + self.add_file(self.builder.src.join(file), destdir.as_ref(), 0o644); + } + } + + pub(crate) fn add_dir(&self, src: impl AsRef, dest: impl AsRef) { + let dest = self.image_dir.join(dest.as_ref()); + + t!(std::fs::create_dir_all(&dest)); + self.builder.cp_r(src.as_ref(), &dest); + } + + pub(crate) fn add_bulk_dir(&mut self, src: impl AsRef, dest: impl AsRef) { + self.bulk_dirs.push(dest.as_ref().to_path_buf()); + self.add_dir(src, dest); + } + + pub(crate) fn generate(self) -> GeneratedTarball { + let mut component_name = self.component.clone(); + if self.is_preview { + component_name.push_str("-preview"); + } + if self.include_target_in_component_name { + component_name.push('-'); + component_name.push_str( + &self + .target + .as_ref() + .expect("include_target_in_component_name used in a targetless tarball"), + ); + } + + self.run(|this, cmd| { + cmd.arg("generate") + .arg("--image-dir") + .arg(&this.image_dir) + .arg(format!("--component-name={}", &component_name)); + + if let Some((dir, dirs)) = this.bulk_dirs.split_first() { + let mut arg = dir.as_os_str().to_os_string(); + for dir in dirs { + arg.push(","); + arg.push(dir); + } + cmd.arg("--bulk-dirs").arg(&arg); + } + + this.non_bare_args(cmd); + }) + } + + pub(crate) fn combine(self, tarballs: &[GeneratedTarball]) -> GeneratedTarball { + let mut input_tarballs = tarballs[0].path.as_os_str().to_os_string(); + for tarball in &tarballs[1..] { + input_tarballs.push(","); + input_tarballs.push(&tarball.path); + } + + self.run(|this, cmd| { + cmd.arg("combine").arg("--input-tarballs").arg(input_tarballs); + this.non_bare_args(cmd); + }) + } + + pub(crate) fn bare(self) -> GeneratedTarball { + // Bare tarballs should have the top level directory match the package + // name, not "image". We rename the image directory just before passing + // into rust-installer. + let dest = self.temp_dir.join(self.package_name()); + t!(std::fs::rename(&self.image_dir, &dest)); + + self.run(|this, cmd| { + let distdir = distdir(this.builder); + t!(std::fs::create_dir_all(&distdir)); + cmd.arg("tarball") + .arg("--input") + .arg(&dest) + .arg("--output") + .arg(distdir.join(this.package_name())); + }) + } + + fn package_name(&self) -> String { + if let Some(target) = &self.target { + format!("{}-{}", self.pkgname, target) + } else { + self.pkgname.clone() + } + } + + fn non_bare_args(&self, cmd: &mut Command) { + cmd.arg("--rel-manifest-dir=rustlib") + .arg("--legacy-manifest-dirs=rustlib,cargo") + .arg(format!("--product-name={}", self.product_name)) + .arg(format!("--success-message={} installed.", self.component)) + .arg(format!("--package-name={}", self.package_name())) + .arg("--non-installed-overlay") + .arg(&self.overlay_dir) + .arg("--output-dir") + .arg(distdir(self.builder)); + } + + fn run(self, build_cli: impl FnOnce(&Tarball<'a>, &mut Command)) -> GeneratedTarball { + t!(std::fs::create_dir_all(&self.overlay_dir)); + self.builder.create(&self.overlay_dir.join("version"), &self.overlay.version(self.builder)); + if let Some(info) = self.builder.rust_info().info() { + channel::write_commit_hash_file(&self.overlay_dir, &info.sha); + channel::write_commit_info_file(&self.overlay_dir, info); + } + for file in self.overlay.legal_and_readme() { + self.builder.install(&self.builder.src.join(file), &self.overlay_dir, 0o644); + } + + let mut cmd = self.builder.tool_cmd(crate::core::build_steps::tool::Tool::RustInstaller); + + let package_name = self.package_name(); + self.builder.info(&format!("Dist {package_name}")); + let _time = crate::utils::helpers::timeit(self.builder); + + build_cli(&self, &mut cmd); + cmd.arg("--work-dir").arg(&self.temp_dir); + if let Some(formats) = &self.builder.config.dist_compression_formats { + assert!(!formats.is_empty(), "dist.compression-formats can't be empty"); + cmd.arg("--compression-formats").arg(formats.join(",")); + } + cmd.args(&["--compression-profile", &self.builder.config.dist_compression_profile]); + self.builder.run(&mut cmd); + + // Ensure there are no symbolic links in the tarball. In particular, + // rustup-toolchain-install-master and most versions of Windows can't handle symbolic links. + let decompressed_output = self.temp_dir.join(&package_name); + if !self.builder.config.dry_run() && !self.permit_symlinks { + for entry in walkdir::WalkDir::new(&decompressed_output) { + let entry = t!(entry); + if entry.path_is_symlink() { + panic!("generated a symlink in a tarball: {}", entry.path().display()); + } + } + } + + // Use either the first compression format defined, or "gz" as the default. + let ext = self + .builder + .config + .dist_compression_formats + .as_ref() + .and_then(|formats| formats.get(0)) + .map(|s| s.as_str()) + .unwrap_or("gz"); + + GeneratedTarball { + path: distdir(self.builder).join(format!("{package_name}.tar.{ext}")), + decompressed_output, + work: self.temp_dir, + } + } +} + +#[derive(Debug, Clone)] +pub struct GeneratedTarball { + path: PathBuf, + decompressed_output: PathBuf, + work: PathBuf, +} + +impl GeneratedTarball { + pub(crate) fn tarball(&self) -> &Path { + &self.path + } + + pub(crate) fn decompressed_output(&self) -> &Path { + &self.decompressed_output + } + + pub(crate) fn work_dir(&self) -> &Path { + &self.work + } +} diff --git a/src/bootstrap/suggest.rs b/src/bootstrap/suggest.rs deleted file mode 100644 index f225104bd..000000000 --- a/src/bootstrap/suggest.rs +++ /dev/null @@ -1,74 +0,0 @@ -#![cfg_attr(feature = "build-metrics", allow(unused))] - -use std::str::FromStr; - -use std::path::PathBuf; - -use clap::Parser; - -use crate::{builder::Builder, tool::Tool}; - -/// Suggests a list of possible `x.py` commands to run based on modified files in branch. -pub fn suggest(builder: &Builder<'_>, run: bool) { - let suggestions = - builder.tool_cmd(Tool::SuggestTests).output().expect("failed to run `suggest-tests` tool"); - - if !suggestions.status.success() { - println!("failed to run `suggest-tests` tool ({})", suggestions.status); - println!( - "`suggest_tests` stdout:\n{}`suggest_tests` stderr:\n{}", - String::from_utf8(suggestions.stdout).unwrap(), - String::from_utf8(suggestions.stderr).unwrap() - ); - panic!("failed to run `suggest-tests`"); - } - - let suggestions = String::from_utf8(suggestions.stdout).unwrap(); - let suggestions = suggestions - .lines() - .map(|line| { - let mut sections = line.split_ascii_whitespace(); - - // this code expects one suggestion per line in the following format: - // {some number of flags} [optional stage number] - let cmd = sections.next().unwrap(); - let stage = sections.next_back().map(|s| str::parse(s).ok()).flatten(); - let paths: Vec = sections.map(|p| PathBuf::from_str(p).unwrap()).collect(); - - (cmd, stage, paths) - }) - .collect::>(); - - if !suggestions.is_empty() { - println!("==== SUGGESTIONS ===="); - for sug in &suggestions { - print!("x {} ", sug.0); - if let Some(stage) = sug.1 { - print!("--stage {stage} "); - } - - for path in &sug.2 { - print!("{} ", path.display()); - } - println!(); - } - println!("====================="); - } else { - println!("No suggestions found!"); - return; - } - - if run { - for sug in suggestions { - let mut build: crate::Build = builder.build.clone(); - build.config.paths = sug.2; - build.config.cmd = crate::flags::Flags::parse_from(["x.py", sug.0]).cmd; - if let Some(stage) = sug.1 { - build.config.stage = stage; - } - build.build(); - } - } else { - println!("help: consider using the `--run` flag to automatically run suggested tests"); - } -} diff --git a/src/bootstrap/synthetic_targets.rs b/src/bootstrap/synthetic_targets.rs deleted file mode 100644 index 7eeac9025..000000000 --- a/src/bootstrap/synthetic_targets.rs +++ /dev/null @@ -1,82 +0,0 @@ -//! In some cases, parts of bootstrap need to change part of a target spec just for one or a few -//! steps. Adding these targets to rustc proper would "leak" this implementation detail of -//! bootstrap, and would make it more complex to apply additional changes if the need arises. -//! -//! To address that problem, this module implements support for "synthetic targets". Synthetic -//! targets are custom target specs generated using builtin target specs as their base. You can use -//! one of the target specs already defined in this module, or create new ones by adding a new step -//! that calls create_synthetic_target. - -use crate::builder::{Builder, ShouldRun, Step}; -use crate::config::TargetSelection; -use crate::Compiler; -use std::process::{Command, Stdio}; - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub(crate) struct MirOptPanicAbortSyntheticTarget { - pub(crate) compiler: Compiler, - pub(crate) base: TargetSelection, -} - -impl Step for MirOptPanicAbortSyntheticTarget { - type Output = TargetSelection; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - create_synthetic_target(builder, self.compiler, "miropt-abort", self.base, |spec| { - spec.insert("panic-strategy".into(), "abort".into()); - }) - } -} - -fn create_synthetic_target( - builder: &Builder<'_>, - compiler: Compiler, - suffix: &str, - base: TargetSelection, - customize: impl FnOnce(&mut serde_json::Map), -) -> TargetSelection { - if base.contains("synthetic") { - // This check is not strictly needed, but nothing currently needs recursive synthetic - // targets. If the need arises, removing this in the future *SHOULD* be safe. - panic!("cannot create synthetic targets with other synthetic targets as their base"); - } - - let name = format!("{base}-synthetic-{suffix}"); - let path = builder.out.join("synthetic-target-specs").join(format!("{name}.json")); - std::fs::create_dir_all(path.parent().unwrap()).unwrap(); - - if builder.config.dry_run() { - std::fs::write(&path, b"dry run\n").unwrap(); - return TargetSelection::create_synthetic(&name, path.to_str().unwrap()); - } - - let mut cmd = Command::new(builder.rustc(compiler)); - cmd.arg("--target").arg(base.rustc_target_arg()); - cmd.args(["-Zunstable-options", "--print", "target-spec-json"]); - cmd.stdout(Stdio::piped()); - - let output = cmd.spawn().unwrap().wait_with_output().unwrap(); - if !output.status.success() { - panic!("failed to gather the target spec for {base}"); - } - - let mut spec: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); - let spec_map = spec.as_object_mut().unwrap(); - - // The `is-builtin` attribute of a spec needs to be removed, otherwise rustc will complain. - spec_map.remove("is-builtin"); - - customize(spec_map); - - std::fs::write(&path, &serde_json::to_vec_pretty(&spec).unwrap()).unwrap(); - let target = TargetSelection::create_synthetic(&name, path.to_str().unwrap()); - crate::cc_detect::find_target(builder, target); - - target -} diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs deleted file mode 100644 index 95d909c57..000000000 --- a/src/bootstrap/tarball.rs +++ /dev/null @@ -1,373 +0,0 @@ -use std::{ - path::{Path, PathBuf}, - process::Command, -}; - -use crate::builder::Builder; -use crate::channel; -use crate::util::t; - -#[derive(Copy, Clone)] -pub(crate) enum OverlayKind { - Rust, - LLVM, - Cargo, - Clippy, - Miri, - Rustfmt, - RustDemangler, - RLS, - RustAnalyzer, -} - -impl OverlayKind { - fn legal_and_readme(&self) -> &[&str] { - match self { - OverlayKind::Rust => &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"], - OverlayKind::LLVM => { - &["src/llvm-project/llvm/LICENSE.TXT", "src/llvm-project/llvm/README.txt"] - } - OverlayKind::Cargo => &[ - "src/tools/cargo/README.md", - "src/tools/cargo/LICENSE-MIT", - "src/tools/cargo/LICENSE-APACHE", - "src/tools/cargo/LICENSE-THIRD-PARTY", - ], - OverlayKind::Clippy => &[ - "src/tools/clippy/README.md", - "src/tools/clippy/LICENSE-APACHE", - "src/tools/clippy/LICENSE-MIT", - ], - OverlayKind::Miri => &[ - "src/tools/miri/README.md", - "src/tools/miri/LICENSE-APACHE", - "src/tools/miri/LICENSE-MIT", - ], - OverlayKind::Rustfmt => &[ - "src/tools/rustfmt/README.md", - "src/tools/rustfmt/LICENSE-APACHE", - "src/tools/rustfmt/LICENSE-MIT", - ], - OverlayKind::RustDemangler => { - &["src/tools/rust-demangler/README.md", "LICENSE-APACHE", "LICENSE-MIT"] - } - OverlayKind::RLS => &["src/tools/rls/README.md", "LICENSE-APACHE", "LICENSE-MIT"], - OverlayKind::RustAnalyzer => &[ - "src/tools/rust-analyzer/README.md", - "src/tools/rust-analyzer/LICENSE-APACHE", - "src/tools/rust-analyzer/LICENSE-MIT", - ], - } - } - - fn version(&self, builder: &Builder<'_>) -> String { - match self { - OverlayKind::Rust => builder.rust_version(), - OverlayKind::LLVM => builder.rust_version(), - OverlayKind::RustDemangler => builder.release_num("rust-demangler"), - OverlayKind::Cargo => { - builder.cargo_info.version(builder, &builder.release_num("cargo")) - } - OverlayKind::Clippy => { - builder.clippy_info.version(builder, &builder.release_num("clippy")) - } - OverlayKind::Miri => builder.miri_info.version(builder, &builder.release_num("miri")), - OverlayKind::Rustfmt => { - builder.rustfmt_info.version(builder, &builder.release_num("rustfmt")) - } - OverlayKind::RLS => builder.release(&builder.release_num("rls")), - OverlayKind::RustAnalyzer => builder - .rust_analyzer_info - .version(builder, &builder.release_num("rust-analyzer/crates/rust-analyzer")), - } - } -} - -pub(crate) struct Tarball<'a> { - builder: &'a Builder<'a>, - - pkgname: String, - component: String, - target: Option, - product_name: String, - overlay: OverlayKind, - - temp_dir: PathBuf, - image_dir: PathBuf, - overlay_dir: PathBuf, - bulk_dirs: Vec, - - include_target_in_component_name: bool, - is_preview: bool, - permit_symlinks: bool, -} - -impl<'a> Tarball<'a> { - pub(crate) fn new(builder: &'a Builder<'a>, component: &str, target: &str) -> Self { - Self::new_inner(builder, component, Some(target.into())) - } - - pub(crate) fn new_targetless(builder: &'a Builder<'a>, component: &str) -> Self { - Self::new_inner(builder, component, None) - } - - fn new_inner(builder: &'a Builder<'a>, component: &str, target: Option) -> Self { - let pkgname = crate::dist::pkgname(builder, component); - - let mut temp_dir = builder.out.join("tmp").join("tarball").join(component); - if let Some(target) = &target { - temp_dir = temp_dir.join(target); - } - let _ = std::fs::remove_dir_all(&temp_dir); - - let image_dir = temp_dir.join("image"); - let overlay_dir = temp_dir.join("overlay"); - - Self { - builder, - - pkgname, - component: component.into(), - target, - product_name: "Rust".into(), - overlay: OverlayKind::Rust, - - temp_dir, - image_dir, - overlay_dir, - bulk_dirs: Vec::new(), - - include_target_in_component_name: false, - is_preview: false, - permit_symlinks: false, - } - } - - pub(crate) fn set_overlay(&mut self, overlay: OverlayKind) { - self.overlay = overlay; - } - - pub(crate) fn set_product_name(&mut self, name: &str) { - self.product_name = name.into(); - } - - pub(crate) fn include_target_in_component_name(&mut self, include: bool) { - self.include_target_in_component_name = include; - } - - pub(crate) fn is_preview(&mut self, is: bool) { - self.is_preview = is; - } - - pub(crate) fn permit_symlinks(&mut self, flag: bool) { - self.permit_symlinks = flag; - } - - pub(crate) fn image_dir(&self) -> &Path { - t!(std::fs::create_dir_all(&self.image_dir)); - &self.image_dir - } - - pub(crate) fn add_file(&self, src: impl AsRef, destdir: impl AsRef, perms: u32) { - // create_dir_all fails to create `foo/bar/.`, so when the destination is "." this simply - // uses the base directory as the destination directory. - let destdir = if destdir.as_ref() == Path::new(".") { - self.image_dir.clone() - } else { - self.image_dir.join(destdir.as_ref()) - }; - - t!(std::fs::create_dir_all(&destdir)); - self.builder.install(src.as_ref(), &destdir, perms); - } - - pub(crate) fn add_renamed_file( - &self, - src: impl AsRef, - destdir: impl AsRef, - new_name: &str, - ) { - let destdir = self.image_dir.join(destdir.as_ref()); - t!(std::fs::create_dir_all(&destdir)); - self.builder.copy(src.as_ref(), &destdir.join(new_name)); - } - - pub(crate) fn add_legal_and_readme_to(&self, destdir: impl AsRef) { - for file in self.overlay.legal_and_readme() { - self.add_file(self.builder.src.join(file), destdir.as_ref(), 0o644); - } - } - - pub(crate) fn add_dir(&self, src: impl AsRef, dest: impl AsRef) { - let dest = self.image_dir.join(dest.as_ref()); - - t!(std::fs::create_dir_all(&dest)); - self.builder.cp_r(src.as_ref(), &dest); - } - - pub(crate) fn add_bulk_dir(&mut self, src: impl AsRef, dest: impl AsRef) { - self.bulk_dirs.push(dest.as_ref().to_path_buf()); - self.add_dir(src, dest); - } - - pub(crate) fn generate(self) -> GeneratedTarball { - let mut component_name = self.component.clone(); - if self.is_preview { - component_name.push_str("-preview"); - } - if self.include_target_in_component_name { - component_name.push('-'); - component_name.push_str( - &self - .target - .as_ref() - .expect("include_target_in_component_name used in a targetless tarball"), - ); - } - - self.run(|this, cmd| { - cmd.arg("generate") - .arg("--image-dir") - .arg(&this.image_dir) - .arg(format!("--component-name={}", &component_name)); - - if let Some((dir, dirs)) = this.bulk_dirs.split_first() { - let mut arg = dir.as_os_str().to_os_string(); - for dir in dirs { - arg.push(","); - arg.push(dir); - } - cmd.arg("--bulk-dirs").arg(&arg); - } - - this.non_bare_args(cmd); - }) - } - - pub(crate) fn combine(self, tarballs: &[GeneratedTarball]) -> GeneratedTarball { - let mut input_tarballs = tarballs[0].path.as_os_str().to_os_string(); - for tarball in &tarballs[1..] { - input_tarballs.push(","); - input_tarballs.push(&tarball.path); - } - - self.run(|this, cmd| { - cmd.arg("combine").arg("--input-tarballs").arg(input_tarballs); - this.non_bare_args(cmd); - }) - } - - pub(crate) fn bare(self) -> GeneratedTarball { - // Bare tarballs should have the top level directory match the package - // name, not "image". We rename the image directory just before passing - // into rust-installer. - let dest = self.temp_dir.join(self.package_name()); - t!(std::fs::rename(&self.image_dir, &dest)); - - self.run(|this, cmd| { - let distdir = crate::dist::distdir(this.builder); - t!(std::fs::create_dir_all(&distdir)); - cmd.arg("tarball") - .arg("--input") - .arg(&dest) - .arg("--output") - .arg(distdir.join(this.package_name())); - }) - } - - fn package_name(&self) -> String { - if let Some(target) = &self.target { - format!("{}-{}", self.pkgname, target) - } else { - self.pkgname.clone() - } - } - - fn non_bare_args(&self, cmd: &mut Command) { - cmd.arg("--rel-manifest-dir=rustlib") - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg(format!("--product-name={}", self.product_name)) - .arg(format!("--success-message={} installed.", self.component)) - .arg(format!("--package-name={}", self.package_name())) - .arg("--non-installed-overlay") - .arg(&self.overlay_dir) - .arg("--output-dir") - .arg(crate::dist::distdir(self.builder)); - } - - fn run(self, build_cli: impl FnOnce(&Tarball<'a>, &mut Command)) -> GeneratedTarball { - t!(std::fs::create_dir_all(&self.overlay_dir)); - self.builder.create(&self.overlay_dir.join("version"), &self.overlay.version(self.builder)); - if let Some(info) = self.builder.rust_info().info() { - channel::write_commit_hash_file(&self.overlay_dir, &info.sha); - channel::write_commit_info_file(&self.overlay_dir, info); - } - for file in self.overlay.legal_and_readme() { - self.builder.install(&self.builder.src.join(file), &self.overlay_dir, 0o644); - } - - let mut cmd = self.builder.tool_cmd(crate::tool::Tool::RustInstaller); - - let package_name = self.package_name(); - self.builder.info(&format!("Dist {package_name}")); - let _time = crate::util::timeit(self.builder); - - build_cli(&self, &mut cmd); - cmd.arg("--work-dir").arg(&self.temp_dir); - if let Some(formats) = &self.builder.config.dist_compression_formats { - assert!(!formats.is_empty(), "dist.compression-formats can't be empty"); - cmd.arg("--compression-formats").arg(formats.join(",")); - } - cmd.args(&["--compression-profile", &self.builder.config.dist_compression_profile]); - self.builder.run(&mut cmd); - - // Ensure there are no symbolic links in the tarball. In particular, - // rustup-toolchain-install-master and most versions of Windows can't handle symbolic links. - let decompressed_output = self.temp_dir.join(&package_name); - if !self.builder.config.dry_run() && !self.permit_symlinks { - for entry in walkdir::WalkDir::new(&decompressed_output) { - let entry = t!(entry); - if entry.path_is_symlink() { - panic!("generated a symlink in a tarball: {}", entry.path().display()); - } - } - } - - // Use either the first compression format defined, or "gz" as the default. - let ext = self - .builder - .config - .dist_compression_formats - .as_ref() - .and_then(|formats| formats.get(0)) - .map(|s| s.as_str()) - .unwrap_or("gz"); - - GeneratedTarball { - path: crate::dist::distdir(self.builder).join(format!("{package_name}.tar.{ext}")), - decompressed_output, - work: self.temp_dir, - } - } -} - -#[derive(Debug, Clone)] -pub struct GeneratedTarball { - path: PathBuf, - decompressed_output: PathBuf, - work: PathBuf, -} - -impl GeneratedTarball { - pub(crate) fn tarball(&self) -> &Path { - &self.path - } - - pub(crate) fn decompressed_output(&self) -> &Path { - &self.decompressed_output - } - - pub(crate) fn work_dir(&self) -> &Path { - &self.work - } -} diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs deleted file mode 100644 index ba030f0f5..000000000 --- a/src/bootstrap/test.rs +++ /dev/null @@ -1,3093 +0,0 @@ -//! Implementation of the test-related targets of the build system. -//! -//! This file implements the various regression test suites that we execute on -//! our CI. - -use std::env; -use std::ffi::OsStr; -use std::ffi::OsString; -use std::fs; -use std::iter; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; - -use clap_complete::shells; - -use crate::builder::crate_description; -use crate::builder::{Builder, Compiler, Kind, RunConfig, ShouldRun, Step}; -use crate::cache::Interned; -use crate::cache::INTERNER; -use crate::compile; -use crate::config::TargetSelection; -use crate::dist; -use crate::doc::DocumentationFormat; -use crate::flags::Subcommand; -use crate::llvm; -use crate::render_tests::add_flags_and_try_run_tests; -use crate::synthetic_targets::MirOptPanicAbortSyntheticTarget; -use crate::tool::{self, SourceType, Tool}; -use crate::toolstate::ToolState; -use crate::util::{self, add_link_lib_path, dylib_path, dylib_path_var, output, t, up_to_date}; -use crate::{envify, CLang, DocTests, GitRepo, Mode}; - -const ADB_TEST_DIR: &str = "/data/local/tmp/work"; - -// mir-opt tests have different variants depending on whether a target is 32bit or 64bit, and -// blessing them requires blessing with each target. To aid developers, when blessing the mir-opt -// test suite the corresponding target of the opposite pointer size is also blessed. -// -// This array serves as the known mappings between 32bit and 64bit targets. If you're developing on -// a target where a target with the opposite pointer size exists, feel free to add it here. -const MIR_OPT_BLESS_TARGET_MAPPING: &[(&str, &str)] = &[ - // (32bit, 64bit) - ("i686-unknown-linux-gnu", "x86_64-unknown-linux-gnu"), - ("i686-unknown-linux-musl", "x86_64-unknown-linux-musl"), - ("i686-pc-windows-msvc", "x86_64-pc-windows-msvc"), - ("i686-pc-windows-gnu", "x86_64-pc-windows-gnu"), - ("i686-apple-darwin", "x86_64-apple-darwin"), - // ARM Macs don't have a corresponding 32-bit target that they can (easily) - // build for, so there is no entry for "aarch64-apple-darwin" here. -]; - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct CrateBootstrap { - path: Interned, - host: TargetSelection, -} - -impl Step for CrateBootstrap { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/jsondoclint") - .path("src/tools/suggest-tests") - .path("src/tools/replace-version-placeholder") - .alias("tidyselftest") - } - - fn make_run(run: RunConfig<'_>) { - for path in run.paths { - let path = INTERNER.intern_path(path.assert_single_path().path.clone()); - run.builder.ensure(CrateBootstrap { host: run.target, path }); - } - } - - fn run(self, builder: &Builder<'_>) { - let bootstrap_host = builder.config.build; - let compiler = builder.compiler(0, bootstrap_host); - let mut path = self.path.to_str().unwrap(); - if path == "tidyselftest" { - path = "src/tools/tidy"; - } - - let cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolBootstrap, - bootstrap_host, - "test", - path, - SourceType::InTree, - &[], - ); - let crate_name = path.rsplit_once('/').unwrap().1; - run_cargo_test(cargo, &[], &[], crate_name, crate_name, compiler, bootstrap_host, builder); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Linkcheck { - host: TargetSelection, -} - -impl Step for Linkcheck { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - /// Runs the `linkchecker` tool as compiled in `stage` by the `host` compiler. - /// - /// This tool in `src/tools` will verify the validity of all our links in the - /// documentation to ensure we don't have a bunch of dead ones. - fn run(self, builder: &Builder<'_>) { - let host = self.host; - let hosts = &builder.hosts; - let targets = &builder.targets; - - // if we have different hosts and targets, some things may be built for - // the host (e.g. rustc) and others for the target (e.g. std). The - // documentation built for each will contain broken links to - // docs built for the other platform (e.g. rustc linking to cargo) - if (hosts != targets) && !hosts.is_empty() && !targets.is_empty() { - panic!( - "Linkcheck currently does not support builds with different hosts and targets. -You can skip linkcheck with --skip src/tools/linkchecker" - ); - } - - builder.info(&format!("Linkcheck ({host})")); - - // Test the linkchecker itself. - let bootstrap_host = builder.config.build; - let compiler = builder.compiler(0, bootstrap_host); - - let cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolBootstrap, - bootstrap_host, - "test", - "src/tools/linkchecker", - SourceType::InTree, - &[], - ); - run_cargo_test( - cargo, - &[], - &[], - "linkchecker", - "linkchecker self tests", - compiler, - bootstrap_host, - builder, - ); - - if builder.doc_tests == DocTests::No { - return; - } - - // Build all the default documentation. - builder.default_doc(&[]); - - // Build the linkchecker before calling `msg`, since GHA doesn't support nested groups. - let mut linkchecker = builder.tool_cmd(Tool::Linkchecker); - - // Run the linkchecker. - let _guard = - builder.msg(Kind::Test, compiler.stage, "Linkcheck", bootstrap_host, bootstrap_host); - let _time = util::timeit(&builder); - builder.run_delaying_failure(linkchecker.arg(builder.out.join(host.triple).join("doc"))); - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - let run = run.path("src/tools/linkchecker"); - run.default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Linkcheck { host: run.target }); - } -} - -fn check_if_tidy_is_installed() -> bool { - Command::new("tidy") - .arg("--version") - .stdout(Stdio::null()) - .status() - .map_or(false, |status| status.success()) -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct HtmlCheck { - target: TargetSelection, -} - -impl Step for HtmlCheck { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let run = run.path("src/tools/html-checker"); - run.lazy_default_condition(Box::new(check_if_tidy_is_installed)) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(HtmlCheck { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - if !check_if_tidy_is_installed() { - eprintln!("not running HTML-check tool because `tidy` is missing"); - eprintln!( - "Note that `tidy` is not the in-tree `src/tools/tidy` but needs to be installed" - ); - panic!("Cannot run html-check tests"); - } - // Ensure that a few different kinds of documentation are available. - builder.default_doc(&[]); - builder.ensure(crate::doc::Rustc::new(builder.top_stage, self.target, builder)); - - builder.run_delaying_failure( - builder.tool_cmd(Tool::HtmlChecker).arg(builder.doc_out(self.target)), - ); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Cargotest { - stage: u32, - host: TargetSelection, -} - -impl Step for Cargotest { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/cargotest") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Cargotest { stage: run.builder.top_stage, host: run.target }); - } - - /// Runs the `cargotest` tool as compiled in `stage` by the `host` compiler. - /// - /// This tool in `src/tools` will check out a few Rust projects and run `cargo - /// test` to ensure that we don't regress the test suites there. - fn run(self, builder: &Builder<'_>) { - let compiler = builder.compiler(self.stage, self.host); - builder.ensure(compile::Rustc::new(compiler, compiler.host)); - let cargo = builder.ensure(tool::Cargo { compiler, target: compiler.host }); - - // Note that this is a short, cryptic, and not scoped directory name. This - // is currently to minimize the length of path on Windows where we otherwise - // quickly run into path name limit constraints. - let out_dir = builder.out.join("ct"); - t!(fs::create_dir_all(&out_dir)); - - let _time = util::timeit(&builder); - let mut cmd = builder.tool_cmd(Tool::CargoTest); - builder.run_delaying_failure( - cmd.arg(&cargo) - .arg(&out_dir) - .args(builder.config.test_args()) - .env("RUSTC", builder.rustc(compiler)) - .env("RUSTDOC", builder.rustdoc(compiler)), - ); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Cargo { - stage: u32, - host: TargetSelection, -} - -impl Step for Cargo { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/cargo") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Cargo { stage: run.builder.top_stage, host: run.target }); - } - - /// Runs `cargo test` for `cargo` packaged with Rust. - fn run(self, builder: &Builder<'_>) { - let compiler = builder.compiler(self.stage, self.host); - - builder.ensure(tool::Cargo { compiler, target: self.host }); - let cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - self.host, - "test", - "src/tools/cargo", - SourceType::Submodule, - &[], - ); - - // NOTE: can't use `run_cargo_test` because we need to overwrite `PATH` - let mut cargo = prepare_cargo_test(cargo, &[], &[], "cargo", compiler, self.host, builder); - - // Don't run cross-compile tests, we may not have cross-compiled libstd libs - // available. - cargo.env("CFG_DISABLE_CROSS_TESTS", "1"); - // Forcibly disable tests using nightly features since any changes to - // those features won't be able to land. - cargo.env("CARGO_TEST_DISABLE_NIGHTLY", "1"); - cargo.env("PATH", &path_for_cargo(builder, compiler)); - - #[cfg(feature = "build-metrics")] - builder.metrics.begin_test_suite( - build_helper::metrics::TestSuiteMetadata::CargoPackage { - crates: vec!["cargo".into()], - target: self.host.triple.to_string(), - host: self.host.triple.to_string(), - stage: self.stage, - }, - builder, - ); - - let _time = util::timeit(&builder); - add_flags_and_try_run_tests(builder, &mut cargo); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct RustAnalyzer { - stage: u32, - host: TargetSelection, -} - -impl Step for RustAnalyzer { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/rust-analyzer") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Self { stage: run.builder.top_stage, host: run.target }); - } - - /// Runs `cargo test` for rust-analyzer - fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let host = self.host; - let compiler = builder.compiler(stage, host); - - // We don't need to build the whole Rust Analyzer for the proc-macro-srv test suite, - // but we do need the standard library to be present. - builder.ensure(compile::Std::new(compiler, host)); - - let workspace_path = "src/tools/rust-analyzer"; - // until the whole RA test suite runs on `i686`, we only run - // `proc-macro-srv` tests - let crate_path = "src/tools/rust-analyzer/crates/proc-macro-srv"; - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolStd, - host, - "test", - crate_path, - SourceType::InTree, - &["sysroot-abi".to_owned()], - ); - cargo.allow_features(tool::RustAnalyzer::ALLOW_FEATURES); - - let dir = builder.src.join(workspace_path); - // needed by rust-analyzer to find its own text fixtures, cf. - // https://github.com/rust-analyzer/expect-test/issues/33 - cargo.env("CARGO_WORKSPACE_DIR", &dir); - - // RA's test suite tries to write to the source directory, that can't - // work in Rust CI - cargo.env("SKIP_SLOW_TESTS", "1"); - - cargo.add_rustc_lib_path(builder, compiler); - run_cargo_test(cargo, &[], &[], "rust-analyzer", "rust-analyzer", compiler, host, builder); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Rustfmt { - stage: u32, - host: TargetSelection, -} - -impl Step for Rustfmt { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/rustfmt") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Rustfmt { stage: run.builder.top_stage, host: run.target }); - } - - /// Runs `cargo test` for rustfmt. - fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let host = self.host; - let compiler = builder.compiler(stage, host); - - builder - .ensure(tool::Rustfmt { compiler, target: self.host, extra_features: Vec::new() }) - .expect("in-tree tool"); - - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - host, - "test", - "src/tools/rustfmt", - SourceType::InTree, - &[], - ); - - let dir = testdir(builder, compiler.host); - t!(fs::create_dir_all(&dir)); - cargo.env("RUSTFMT_TEST_DIR", dir); - - cargo.add_rustc_lib_path(builder, compiler); - - run_cargo_test(cargo, &[], &[], "rustfmt", "rustfmt", compiler, host, builder); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct RustDemangler { - stage: u32, - host: TargetSelection, -} - -impl Step for RustDemangler { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/rust-demangler") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustDemangler { stage: run.builder.top_stage, host: run.target }); - } - - /// Runs `cargo test` for rust-demangler. - fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let host = self.host; - let compiler = builder.compiler(stage, host); - - let rust_demangler = builder - .ensure(tool::RustDemangler { compiler, target: self.host, extra_features: Vec::new() }) - .expect("in-tree tool"); - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - host, - "test", - "src/tools/rust-demangler", - SourceType::InTree, - &[], - ); - - let dir = testdir(builder, compiler.host); - t!(fs::create_dir_all(&dir)); - - cargo.env("RUST_DEMANGLER_DRIVER_PATH", rust_demangler); - cargo.add_rustc_lib_path(builder, compiler); - - run_cargo_test( - cargo, - &[], - &[], - "rust-demangler", - "rust-demangler", - compiler, - host, - builder, - ); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Miri { - stage: u32, - host: TargetSelection, - target: TargetSelection, -} - -impl Miri { - /// Run `cargo miri setup` for the given target, return where the Miri sysroot was put. - pub fn build_miri_sysroot( - builder: &Builder<'_>, - compiler: Compiler, - miri: &Path, - target: TargetSelection, - ) -> String { - let miri_sysroot = builder.out.join(compiler.host.triple).join("miri-sysroot"); - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - compiler.host, - "run", - "src/tools/miri/cargo-miri", - SourceType::InTree, - &[], - ); - cargo.add_rustc_lib_path(builder, compiler); - cargo.arg("--").arg("miri").arg("setup"); - cargo.arg("--target").arg(target.rustc_target_arg()); - - // Tell `cargo miri setup` where to find the sources. - cargo.env("MIRI_LIB_SRC", builder.src.join("library")); - // Tell it where to find Miri. - cargo.env("MIRI", &miri); - // Tell it where to put the sysroot. - cargo.env("MIRI_SYSROOT", &miri_sysroot); - // Debug things. - cargo.env("RUST_BACKTRACE", "1"); - - let mut cargo = Command::from(cargo); - let _guard = builder.msg( - Kind::Build, - compiler.stage + 1, - "miri sysroot", - compiler.host, - compiler.host, - ); - builder.run(&mut cargo); - - // # Determine where Miri put its sysroot. - // To this end, we run `cargo miri setup --print-sysroot` and capture the output. - // (We do this separately from the above so that when the setup actually - // happens we get some output.) - // We re-use the `cargo` from above. - cargo.arg("--print-sysroot"); - - // FIXME: Is there a way in which we can re-use the usual `run` helpers? - if builder.config.dry_run() { - String::new() - } else { - builder.verbose(&format!("running: {cargo:?}")); - let out = - cargo.output().expect("We already ran `cargo miri setup` before and that worked"); - assert!(out.status.success(), "`cargo miri setup` returned with non-0 exit code"); - // Output is "\n". - let stdout = String::from_utf8(out.stdout) - .expect("`cargo miri setup` stdout is not valid UTF-8"); - let sysroot = stdout.trim_end(); - builder.verbose(&format!("`cargo miri setup --print-sysroot` said: {sysroot:?}")); - sysroot.to_owned() - } - } -} - -impl Step for Miri { - type Output = (); - const ONLY_HOSTS: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/miri") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Miri { - stage: run.builder.top_stage, - host: run.build_triple(), - target: run.target, - }); - } - - /// Runs `cargo test` for miri. - fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let host = self.host; - let target = self.target; - let compiler = builder.compiler(stage, host); - // We need the stdlib for the *next* stage, as it was built with this compiler that also built Miri. - // Except if we are at stage 2, the bootstrap loop is complete and we can stick with our current stage. - let compiler_std = builder.compiler(if stage < 2 { stage + 1 } else { stage }, host); - - let miri = builder - .ensure(tool::Miri { compiler, target: self.host, extra_features: Vec::new() }) - .expect("in-tree tool"); - let _cargo_miri = builder - .ensure(tool::CargoMiri { compiler, target: self.host, extra_features: Vec::new() }) - .expect("in-tree tool"); - // The stdlib we need might be at a different stage. And just asking for the - // sysroot does not seem to populate it, so we do that first. - builder.ensure(compile::Std::new(compiler_std, host)); - let sysroot = builder.sysroot(compiler_std); - // We also need a Miri sysroot. - let miri_sysroot = Miri::build_miri_sysroot(builder, compiler, &miri, target); - - // # Run `cargo test`. - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - host, - "test", - "src/tools/miri", - SourceType::InTree, - &[], - ); - let _guard = builder.msg_sysroot_tool(Kind::Test, compiler.stage, "miri", host, host); - - cargo.add_rustc_lib_path(builder, compiler); - - // miri tests need to know about the stage sysroot - cargo.env("MIRI_SYSROOT", &miri_sysroot); - cargo.env("MIRI_HOST_SYSROOT", sysroot); - cargo.env("MIRI", &miri); - if builder.config.locked_deps { - // enforce lockfiles - cargo.env("CARGO_EXTRA_FLAGS", "--locked"); - } - - // Set the target. - cargo.env("MIRI_TEST_TARGET", target.rustc_target_arg()); - - // This can NOT be `run_cargo_test` since the Miri test runner - // does not understand the flags added by `add_flags_and_try_run_test`. - let mut cargo = prepare_cargo_test(cargo, &[], &[], "miri", compiler, target, builder); - { - let _time = util::timeit(&builder); - builder.run(&mut cargo); - } - - // Run it again for mir-opt-level 4 to catch some miscompilations. - if builder.config.test_args().is_empty() { - cargo.env("MIRIFLAGS", "-O -Zmir-opt-level=4 -Cdebug-assertions=yes"); - // Optimizations can change backtraces - cargo.env("MIRI_SKIP_UI_CHECKS", "1"); - // `MIRI_SKIP_UI_CHECKS` and `RUSTC_BLESS` are incompatible - cargo.env_remove("RUSTC_BLESS"); - // Optimizations can change error locations and remove UB so don't run `fail` tests. - cargo.args(&["tests/pass", "tests/panic"]); - - let mut cargo = prepare_cargo_test(cargo, &[], &[], "miri", compiler, target, builder); - { - let _time = util::timeit(&builder); - builder.run(&mut cargo); - } - } - - // # Run `cargo miri test`. - // This is just a smoke test (Miri's own CI invokes this in a bunch of different ways and ensures - // that we get the desired output), but that is sufficient to make sure that the libtest harness - // itself executes properly under Miri. - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - host, - "run", - "src/tools/miri/cargo-miri", - SourceType::Submodule, - &[], - ); - cargo.add_rustc_lib_path(builder, compiler); - cargo.arg("--").arg("miri").arg("test"); - if builder.config.locked_deps { - cargo.arg("--locked"); - } - cargo - .arg("--manifest-path") - .arg(builder.src.join("src/tools/miri/test-cargo-miri/Cargo.toml")); - cargo.arg("--target").arg(target.rustc_target_arg()); - cargo.arg("--tests"); // don't run doctests, they are too confused by the staging - cargo.arg("--").args(builder.config.test_args()); - - // Tell `cargo miri` where to find things. - cargo.env("MIRI_SYSROOT", &miri_sysroot); - cargo.env("MIRI_HOST_SYSROOT", sysroot); - cargo.env("MIRI", &miri); - // Debug things. - cargo.env("RUST_BACKTRACE", "1"); - - let mut cargo = Command::from(cargo); - { - let _time = util::timeit(&builder); - builder.run(&mut cargo); - } - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct CompiletestTest { - host: TargetSelection, -} - -impl Step for CompiletestTest { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/compiletest") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(CompiletestTest { host: run.target }); - } - - /// Runs `cargo test` for compiletest. - fn run(self, builder: &Builder<'_>) { - let host = self.host; - let compiler = builder.compiler(builder.top_stage, host); - - // We need `ToolStd` for the locally-built sysroot because - // compiletest uses unstable features of the `test` crate. - builder.ensure(compile::Std::new(compiler, host)); - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolStd, - host, - "test", - "src/tools/compiletest", - SourceType::InTree, - &[], - ); - cargo.allow_features("test"); - run_cargo_test( - cargo, - &[], - &[], - "compiletest", - "compiletest self test", - compiler, - host, - builder, - ); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Clippy { - stage: u32, - host: TargetSelection, -} - -impl Step for Clippy { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/clippy") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Clippy { stage: run.builder.top_stage, host: run.target }); - } - - /// Runs `cargo test` for clippy. - fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let host = self.host; - let compiler = builder.compiler(stage, host); - - builder - .ensure(tool::Clippy { compiler, target: self.host, extra_features: Vec::new() }) - .expect("in-tree tool"); - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - host, - "test", - "src/tools/clippy", - SourceType::InTree, - &[], - ); - - cargo.env("RUSTC_TEST_SUITE", builder.rustc(compiler)); - cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler)); - let host_libs = builder.stage_out(compiler, Mode::ToolRustc).join(builder.cargo_dir()); - cargo.env("HOST_LIBS", host_libs); - - cargo.add_rustc_lib_path(builder, compiler); - let mut cargo = prepare_cargo_test(cargo, &[], &[], "clippy", compiler, host, builder); - - let _guard = builder.msg_sysroot_tool(Kind::Test, compiler.stage, "clippy", host, host); - - #[allow(deprecated)] // Clippy reports errors if it blessed the outputs - if builder.config.try_run(&mut cargo).is_ok() { - // The tests succeeded; nothing to do. - return; - } - - if !builder.config.cmd.bless() { - crate::exit!(1); - } - } -} - -fn path_for_cargo(builder: &Builder<'_>, compiler: Compiler) -> OsString { - // Configure PATH to find the right rustc. NB. we have to use PATH - // and not RUSTC because the Cargo test suite has tests that will - // fail if rustc is not spelled `rustc`. - let path = builder.sysroot(compiler).join("bin"); - let old_path = env::var_os("PATH").unwrap_or_default(); - env::join_paths(iter::once(path).chain(env::split_paths(&old_path))).expect("") -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct RustdocTheme { - pub compiler: Compiler, -} - -impl Step for RustdocTheme { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/rustdoc-themes") - } - - fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.target); - - run.builder.ensure(RustdocTheme { compiler }); - } - - fn run(self, builder: &Builder<'_>) { - let rustdoc = builder.bootstrap_out.join("rustdoc"); - let mut cmd = builder.tool_cmd(Tool::RustdocTheme); - cmd.arg(rustdoc.to_str().unwrap()) - .arg(builder.src.join("src/librustdoc/html/static/css/rustdoc.css").to_str().unwrap()) - .env("RUSTC_STAGE", self.compiler.stage.to_string()) - .env("RUSTC_SYSROOT", builder.sysroot(self.compiler)) - .env("RUSTDOC_LIBDIR", builder.sysroot_libdir(self.compiler, self.compiler.host)) - .env("CFG_RELEASE_CHANNEL", &builder.config.channel) - .env("RUSTDOC_REAL", builder.rustdoc(self.compiler)) - .env("RUSTC_BOOTSTRAP", "1"); - if let Some(linker) = builder.linker(self.compiler.host) { - cmd.env("RUSTDOC_LINKER", linker); - } - if builder.is_fuse_ld_lld(self.compiler.host) { - cmd.env( - "RUSTDOC_LLD_NO_THREADS", - util::lld_flag_no_threads(self.compiler.host.contains("windows")), - ); - } - builder.run_delaying_failure(&mut cmd); - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct RustdocJSStd { - pub target: TargetSelection, -} - -impl Step for RustdocJSStd { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = run.builder.config.nodejs.is_some(); - run.suite_path("tests/rustdoc-js-std").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustdocJSStd { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let nodejs = - builder.config.nodejs.as_ref().expect("need nodejs to run rustdoc-js-std tests"); - let mut command = Command::new(nodejs); - command - .arg(builder.src.join("src/tools/rustdoc-js/tester.js")) - .arg("--crate-name") - .arg("std") - .arg("--resource-suffix") - .arg(&builder.version) - .arg("--doc-folder") - .arg(builder.doc_out(self.target)) - .arg("--test-folder") - .arg(builder.src.join("tests/rustdoc-js-std")); - for path in &builder.paths { - if let Some(p) = util::is_valid_test_suite_arg(path, "tests/rustdoc-js-std", builder) { - if !p.ends_with(".js") { - eprintln!("A non-js file was given: `{}`", path.display()); - panic!("Cannot run rustdoc-js-std tests"); - } - command.arg("--test-file").arg(path); - } - } - builder.ensure(crate::doc::Std::new( - builder.top_stage, - self.target, - builder, - DocumentationFormat::HTML, - )); - let _guard = builder.msg( - Kind::Test, - builder.top_stage, - "rustdoc-js-std", - builder.config.build, - self.target, - ); - builder.run(&mut command); - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct RustdocJSNotStd { - pub target: TargetSelection, - pub compiler: Compiler, -} - -impl Step for RustdocJSNotStd { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = run.builder.config.nodejs.is_some(); - run.suite_path("tests/rustdoc-js").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - run.builder.ensure(RustdocJSNotStd { target: run.target, compiler }); - } - - fn run(self, builder: &Builder<'_>) { - builder.ensure(Compiletest { - compiler: self.compiler, - target: self.target, - mode: "js-doc-test", - suite: "rustdoc-js", - path: "tests/rustdoc-js", - compare_mode: None, - }); - } -} - -fn get_browser_ui_test_version_inner(npm: &Path, global: bool) -> Option { - let mut command = Command::new(&npm); - command.arg("list").arg("--parseable").arg("--long").arg("--depth=0"); - if global { - command.arg("--global"); - } - let lines = command - .output() - .map(|output| String::from_utf8_lossy(&output.stdout).into_owned()) - .unwrap_or(String::new()); - lines - .lines() - .find_map(|l| l.split(':').nth(1)?.strip_prefix("browser-ui-test@")) - .map(|v| v.to_owned()) -} - -fn get_browser_ui_test_version(npm: &Path) -> Option { - get_browser_ui_test_version_inner(npm, false) - .or_else(|| get_browser_ui_test_version_inner(npm, true)) -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct RustdocGUI { - pub target: TargetSelection, - pub compiler: Compiler, -} - -impl Step for RustdocGUI { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - let run = run.suite_path("tests/rustdoc-gui"); - run.lazy_default_condition(Box::new(move || { - builder.config.nodejs.is_some() - && builder - .config - .npm - .as_ref() - .map(|p| get_browser_ui_test_version(p).is_some()) - .unwrap_or(false) - })) - } - - fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - run.builder.ensure(RustdocGUI { target: run.target, compiler }); - } - - fn run(self, builder: &Builder<'_>) { - builder.ensure(compile::Std::new(self.compiler, self.target)); - - let mut cmd = builder.tool_cmd(Tool::RustdocGUITest); - - let out_dir = builder.test_out(self.target).join("rustdoc-gui"); - builder.clear_if_dirty(&out_dir, &builder.rustdoc(self.compiler)); - - if let Some(src) = builder.config.src.to_str() { - cmd.arg("--rust-src").arg(src); - } - - if let Some(out_dir) = out_dir.to_str() { - cmd.arg("--out-dir").arg(out_dir); - } - - if let Some(initial_cargo) = builder.config.initial_cargo.to_str() { - cmd.arg("--initial-cargo").arg(initial_cargo); - } - - cmd.arg("--jobs").arg(builder.jobs().to_string()); - - cmd.env("RUSTDOC", builder.rustdoc(self.compiler)) - .env("RUSTC", builder.rustc(self.compiler)); - - for path in &builder.paths { - if let Some(p) = util::is_valid_test_suite_arg(path, "tests/rustdoc-gui", builder) { - if !p.ends_with(".goml") { - eprintln!("A non-goml file was given: `{}`", path.display()); - panic!("Cannot run rustdoc-gui tests"); - } - if let Some(name) = path.file_name().and_then(|f| f.to_str()) { - cmd.arg("--goml-file").arg(name); - } - } - } - - for test_arg in builder.config.test_args() { - cmd.arg("--test-arg").arg(test_arg); - } - - if let Some(ref nodejs) = builder.config.nodejs { - cmd.arg("--nodejs").arg(nodejs); - } - - if let Some(ref npm) = builder.config.npm { - cmd.arg("--npm").arg(npm); - } - - let _time = util::timeit(&builder); - let _guard = builder.msg_sysroot_tool( - Kind::Test, - self.compiler.stage, - "rustdoc-gui", - self.compiler.host, - self.target, - ); - crate::render_tests::try_run_tests(builder, &mut cmd, true); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Tidy; - -impl Step for Tidy { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - /// Runs the `tidy` tool. - /// - /// This tool in `src/tools` checks up on various bits and pieces of style and - /// otherwise just implements a few lint-like checks that are specific to the - /// compiler itself. - /// - /// Once tidy passes, this step also runs `fmt --check` if tests are being run - /// for the `dev` or `nightly` channels. - fn run(self, builder: &Builder<'_>) { - let mut cmd = builder.tool_cmd(Tool::Tidy); - cmd.arg(&builder.src); - cmd.arg(&builder.initial_cargo); - cmd.arg(&builder.out); - // Tidy is heavily IO constrained. Still respect `-j`, but use a higher limit if `jobs` hasn't been configured. - let jobs = builder.config.jobs.unwrap_or_else(|| { - 8 * std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32 - }); - cmd.arg(jobs.to_string()); - if builder.is_verbose() { - cmd.arg("--verbose"); - } - if builder.config.cmd.bless() { - cmd.arg("--bless"); - } - if let Some(s) = builder.config.cmd.extra_checks() { - cmd.arg(format!("--extra-checks={s}")); - } - let mut args = std::env::args_os(); - if let Some(_) = args.find(|arg| arg == OsStr::new("--")) { - cmd.arg("--"); - cmd.args(args); - } - - if builder.config.channel == "dev" || builder.config.channel == "nightly" { - builder.info("fmt check"); - if builder.initial_rustfmt().is_none() { - let inferred_rustfmt_dir = builder.initial_rustc.parent().unwrap(); - eprintln!( - "\ -error: no `rustfmt` binary found in {PATH} -info: `rust.channel` is currently set to \"{CHAN}\" -help: if you are testing a beta branch, set `rust.channel` to \"beta\" in the `config.toml` file -help: to skip test's attempt to check tidiness, pass `--skip src/tools/tidy` to `x.py test`", - PATH = inferred_rustfmt_dir.display(), - CHAN = builder.config.channel, - ); - crate::exit!(1); - } - crate::format::format(&builder, !builder.config.cmd.bless(), &[]); - } - - builder.info("tidy check"); - builder.run_delaying_failure(&mut cmd); - - builder.ensure(ExpandYamlAnchors); - - builder.info("x.py completions check"); - let [bash, fish, powershell] = ["x.py.sh", "x.py.fish", "x.py.ps1"] - .map(|filename| builder.src.join("src/etc/completions").join(filename)); - if builder.config.cmd.bless() { - builder.ensure(crate::run::GenerateCompletions); - } else if crate::flags::get_completion(shells::Bash, &bash).is_some() - || crate::flags::get_completion(shells::Fish, &fish).is_some() - || crate::flags::get_completion(shells::PowerShell, &powershell).is_some() - { - eprintln!( - "x.py completions were changed; run `x.py run generate-completions` to update them" - ); - crate::exit!(1); - } - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/tidy") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Tidy); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct ExpandYamlAnchors; - -impl Step for ExpandYamlAnchors { - type Output = (); - const ONLY_HOSTS: bool = true; - - /// Ensure the `generate-ci-config` tool was run locally. - /// - /// The tool in `src/tools` reads the CI definition in `src/ci/builders.yml` and generates the - /// appropriate configuration for all our CI providers. This step ensures the tool was called - /// by the user before committing CI changes. - fn run(self, builder: &Builder<'_>) { - // Note: `.github/` is not included in dist-src tarballs - if !builder.src.join(".github/workflows/ci.yml").exists() { - builder.info("Skipping YAML anchors check: GitHub Actions config not found"); - return; - } - builder.info("Ensuring the YAML anchors in the GitHub Actions config were expanded"); - builder.run_delaying_failure( - &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("check").arg(&builder.src), - ); - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/expand-yaml-anchors") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(ExpandYamlAnchors); - } -} - -fn testdir(builder: &Builder<'_>, host: TargetSelection) -> PathBuf { - builder.out.join(host.triple).join("test") -} - -macro_rules! default_test { - ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr }) => { - test!($name { path: $path, mode: $mode, suite: $suite, default: true, host: false }); - }; -} - -macro_rules! default_test_with_compare_mode { - ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, - compare_mode: $compare_mode:expr }) => { - test_with_compare_mode!($name { - path: $path, - mode: $mode, - suite: $suite, - default: true, - host: false, - compare_mode: $compare_mode - }); - }; -} - -macro_rules! host_test { - ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr }) => { - test!($name { path: $path, mode: $mode, suite: $suite, default: true, host: true }); - }; -} - -macro_rules! test { - ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, default: $default:expr, - host: $host:expr }) => { - test_definitions!($name { - path: $path, - mode: $mode, - suite: $suite, - default: $default, - host: $host, - compare_mode: None - }); - }; -} - -macro_rules! test_with_compare_mode { - ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, default: $default:expr, - host: $host:expr, compare_mode: $compare_mode:expr }) => { - test_definitions!($name { - path: $path, - mode: $mode, - suite: $suite, - default: $default, - host: $host, - compare_mode: Some($compare_mode) - }); - }; -} - -macro_rules! test_definitions { - ($name:ident { - path: $path:expr, - mode: $mode:expr, - suite: $suite:expr, - default: $default:expr, - host: $host:expr, - compare_mode: $compare_mode:expr - }) => { - #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] - pub struct $name { - pub compiler: Compiler, - pub target: TargetSelection, - } - - impl Step for $name { - type Output = (); - const DEFAULT: bool = $default; - const ONLY_HOSTS: bool = $host; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.suite_path($path) - } - - fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - - run.builder.ensure($name { compiler, target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - builder.ensure(Compiletest { - compiler: self.compiler, - target: self.target, - mode: $mode, - suite: $suite, - path: $path, - compare_mode: $compare_mode, - }) - } - } - }; -} - -default_test!(Ui { path: "tests/ui", mode: "ui", suite: "ui" }); - -default_test!(RunPassValgrind { - path: "tests/run-pass-valgrind", - mode: "run-pass-valgrind", - suite: "run-pass-valgrind" -}); - -default_test!(Codegen { path: "tests/codegen", mode: "codegen", suite: "codegen" }); - -default_test!(CodegenUnits { - path: "tests/codegen-units", - mode: "codegen-units", - suite: "codegen-units" -}); - -default_test!(Incremental { path: "tests/incremental", mode: "incremental", suite: "incremental" }); - -default_test_with_compare_mode!(Debuginfo { - path: "tests/debuginfo", - mode: "debuginfo", - suite: "debuginfo", - compare_mode: "split-dwarf" -}); - -host_test!(UiFullDeps { path: "tests/ui-fulldeps", mode: "ui", suite: "ui-fulldeps" }); - -host_test!(Rustdoc { path: "tests/rustdoc", mode: "rustdoc", suite: "rustdoc" }); -host_test!(RustdocUi { path: "tests/rustdoc-ui", mode: "ui", suite: "rustdoc-ui" }); - -host_test!(RustdocJson { path: "tests/rustdoc-json", mode: "rustdoc-json", suite: "rustdoc-json" }); - -host_test!(Pretty { path: "tests/pretty", mode: "pretty", suite: "pretty" }); - -default_test!(RunMake { path: "tests/run-make", mode: "run-make", suite: "run-make" }); - -host_test!(RunMakeFullDeps { - path: "tests/run-make-fulldeps", - mode: "run-make", - suite: "run-make-fulldeps" -}); - -default_test!(Assembly { path: "tests/assembly", mode: "assembly", suite: "assembly" }); - -default_test!(CoverageMap { - path: "tests/coverage-map", - mode: "coverage-map", - suite: "coverage-map" -}); - -host_test!(RunCoverage { path: "tests/run-coverage", mode: "run-coverage", suite: "run-coverage" }); -host_test!(RunCoverageRustdoc { - path: "tests/run-coverage-rustdoc", - mode: "run-coverage", - suite: "run-coverage-rustdoc" -}); - -// For the mir-opt suite we do not use macros, as we need custom behavior when blessing. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct MirOpt { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for MirOpt { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.suite_path("tests/mir-opt") - } - - fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - run.builder.ensure(MirOpt { compiler, target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let run = |target| { - builder.ensure(Compiletest { - compiler: self.compiler, - target, - mode: "mir-opt", - suite: "mir-opt", - path: "tests/mir-opt", - compare_mode: None, - }) - }; - - // We use custom logic to bless the mir-opt suite: mir-opt tests have multiple variants - // (32bit vs 64bit, and panic=abort vs panic=unwind), and all of them needs to be blessed. - // When blessing, we try best-effort to also bless the other variants, to aid developers. - if builder.config.cmd.bless() { - let targets = MIR_OPT_BLESS_TARGET_MAPPING - .iter() - .filter(|(target_32bit, target_64bit)| { - *target_32bit == &*self.target.triple || *target_64bit == &*self.target.triple - }) - .next() - .map(|(target_32bit, target_64bit)| { - let target_32bit = TargetSelection::from_user(target_32bit); - let target_64bit = TargetSelection::from_user(target_64bit); - - // Running compiletest requires a C compiler to be available, but it might not - // have been detected by bootstrap if the target we're testing wasn't in the - // --target flags. - if !builder.cc.borrow().contains_key(&target_32bit) { - crate::cc_detect::find_target(builder, target_32bit); - } - if !builder.cc.borrow().contains_key(&target_64bit) { - crate::cc_detect::find_target(builder, target_64bit); - } - - vec![target_32bit, target_64bit] - }) - .unwrap_or_else(|| { - eprintln!( - "\ -Note that not all variants of mir-opt tests are going to be blessed, as no mapping between -a 32bit and a 64bit target was found for {target}. -You can add that mapping by changing MIR_OPT_BLESS_TARGET_MAPPING in src/bootstrap/test.rs", - target = self.target, - ); - vec![self.target] - }); - - for target in targets { - run(target); - - let panic_abort_target = builder.ensure(MirOptPanicAbortSyntheticTarget { - compiler: self.compiler, - base: target, - }); - run(panic_abort_target); - } - } else { - run(self.target); - } - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -struct Compiletest { - compiler: Compiler, - target: TargetSelection, - mode: &'static str, - suite: &'static str, - path: &'static str, - compare_mode: Option<&'static str>, -} - -impl Step for Compiletest { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Executes the `compiletest` tool to run a suite of tests. - /// - /// Compiles all tests with `compiler` for `target` with the specified - /// compiletest `mode` and `suite` arguments. For example `mode` can be - /// "run-pass" or `suite` can be something like `debuginfo`. - fn run(self, builder: &Builder<'_>) { - if builder.top_stage == 0 && env::var("COMPILETEST_FORCE_STAGE0").is_err() { - eprintln!("\ -error: `--stage 0` runs compiletest on the beta compiler, not your local changes, and will almost always cause tests to fail -help: to test the compiler, use `--stage 1` instead -help: to test the standard library, use `--stage 0 library/std` instead -note: if you're sure you want to do this, please open an issue as to why. In the meantime, you can override this with `COMPILETEST_FORCE_STAGE0=1`." - ); - crate::exit!(1); - } - - let mut compiler = self.compiler; - let target = self.target; - let mode = self.mode; - let suite = self.suite; - - // Path for test suite - let suite_path = self.path; - - // Skip codegen tests if they aren't enabled in configuration. - if !builder.config.codegen_tests && suite == "codegen" { - return; - } - - // Support stage 1 ui-fulldeps. This is somewhat complicated: ui-fulldeps tests for the most - // part test the *API* of the compiler, not how it compiles a given file. As a result, we - // can run them against the stage 1 sources as long as we build them with the stage 0 - // bootstrap compiler. - // NOTE: Only stage 1 is special cased because we need the rustc_private artifacts to match the - // running compiler in stage 2 when plugins run. - let stage_id = if suite == "ui-fulldeps" && compiler.stage == 1 { - compiler = builder.compiler(compiler.stage - 1, target); - format!("stage{}-{}", compiler.stage + 1, target) - } else { - format!("stage{}-{}", compiler.stage, target) - }; - - if suite.ends_with("fulldeps") { - builder.ensure(compile::Rustc::new(compiler, target)); - } - - if suite == "debuginfo" { - builder - .ensure(dist::DebuggerScripts { sysroot: builder.sysroot(compiler), host: target }); - } - - builder.ensure(compile::Std::new(compiler, target)); - // ensure that `libproc_macro` is available on the host. - builder.ensure(compile::Std::new(compiler, compiler.host)); - - // Also provide `rust_test_helpers` for the host. - builder.ensure(TestHelpers { target: compiler.host }); - - // As well as the target, except for plain wasm32, which can't build it - if !target.contains("wasm") || target.contains("emscripten") { - builder.ensure(TestHelpers { target }); - } - - builder.ensure(RemoteCopyLibs { compiler, target }); - - let mut cmd = builder.tool_cmd(Tool::Compiletest); - - // compiletest currently has... a lot of arguments, so let's just pass all - // of them! - - cmd.arg("--compile-lib-path").arg(builder.rustc_libdir(compiler)); - cmd.arg("--run-lib-path").arg(builder.sysroot_libdir(compiler, target)); - cmd.arg("--rustc-path").arg(builder.rustc(compiler)); - - let is_rustdoc = suite.ends_with("rustdoc-ui") || suite.ends_with("rustdoc-js"); - - // Avoid depending on rustdoc when we don't need it. - if mode == "rustdoc" - || mode == "run-make" - || (mode == "ui" && is_rustdoc) - || mode == "js-doc-test" - || mode == "rustdoc-json" - || suite == "run-coverage-rustdoc" - { - cmd.arg("--rustdoc-path").arg(builder.rustdoc(compiler)); - } - - if mode == "rustdoc-json" { - // Use the beta compiler for jsondocck - let json_compiler = compiler.with_stage(0); - cmd.arg("--jsondocck-path") - .arg(builder.ensure(tool::JsonDocCk { compiler: json_compiler, target })); - cmd.arg("--jsondoclint-path") - .arg(builder.ensure(tool::JsonDocLint { compiler: json_compiler, target })); - } - - if mode == "coverage-map" { - let coverage_dump = builder.ensure(tool::CoverageDump { - compiler: compiler.with_stage(0), - target: compiler.host, - }); - cmd.arg("--coverage-dump-path").arg(coverage_dump); - } - - if mode == "run-make" || mode == "run-coverage" { - let rust_demangler = builder - .ensure(tool::RustDemangler { - compiler, - target: compiler.host, - extra_features: Vec::new(), - }) - .expect("in-tree tool"); - cmd.arg("--rust-demangler-path").arg(rust_demangler); - } - - cmd.arg("--src-base").arg(builder.src.join("tests").join(suite)); - cmd.arg("--build-base").arg(testdir(builder, compiler.host).join(suite)); - - // When top stage is 0, that means that we're testing an externally provided compiler. - // In that case we need to use its specific sysroot for tests to pass. - let sysroot = if builder.top_stage == 0 { - builder.initial_sysroot.clone() - } else { - builder.sysroot(compiler).to_path_buf() - }; - cmd.arg("--sysroot-base").arg(sysroot); - cmd.arg("--stage-id").arg(stage_id); - cmd.arg("--suite").arg(suite); - cmd.arg("--mode").arg(mode); - cmd.arg("--target").arg(target.rustc_target_arg()); - cmd.arg("--host").arg(&*compiler.host.triple); - cmd.arg("--llvm-filecheck").arg(builder.llvm_filecheck(builder.config.build)); - - if builder.config.cmd.bless() { - cmd.arg("--bless"); - } - - if builder.config.cmd.force_rerun() { - cmd.arg("--force-rerun"); - } - - let compare_mode = - builder.config.cmd.compare_mode().or_else(|| { - if builder.config.test_compare_mode { self.compare_mode } else { None } - }); - - if let Some(ref pass) = builder.config.cmd.pass() { - cmd.arg("--pass"); - cmd.arg(pass); - } - - if let Some(ref run) = builder.config.cmd.run() { - cmd.arg("--run"); - cmd.arg(run); - } - - if let Some(ref nodejs) = builder.config.nodejs { - cmd.arg("--nodejs").arg(nodejs); - } else if mode == "js-doc-test" { - panic!("need nodejs to run js-doc-test suite"); - } - if let Some(ref npm) = builder.config.npm { - cmd.arg("--npm").arg(npm); - } - if builder.config.rust_optimize_tests { - cmd.arg("--optimize-tests"); - } - if builder.config.cmd.only_modified() { - cmd.arg("--only-modified"); - } - - let mut flags = if is_rustdoc { Vec::new() } else { vec!["-Crpath".to_string()] }; - flags.push(format!("-Cdebuginfo={}", builder.config.rust_debuginfo_level_tests)); - flags.extend(builder.config.cmd.rustc_args().iter().map(|s| s.to_string())); - - if let Some(linker) = builder.linker(target) { - cmd.arg("--target-linker").arg(linker); - } - if let Some(linker) = builder.linker(compiler.host) { - cmd.arg("--host-linker").arg(linker); - } - - let mut hostflags = flags.clone(); - hostflags.push(format!("-Lnative={}", builder.test_helpers_out(compiler.host).display())); - hostflags.extend(builder.lld_flags(compiler.host)); - for flag in hostflags { - cmd.arg("--host-rustcflags").arg(flag); - } - - let mut targetflags = flags; - targetflags.push(format!("-Lnative={}", builder.test_helpers_out(target).display())); - targetflags.extend(builder.lld_flags(target)); - for flag in targetflags { - cmd.arg("--target-rustcflags").arg(flag); - } - - cmd.arg("--python").arg(builder.python()); - - if let Some(ref gdb) = builder.config.gdb { - cmd.arg("--gdb").arg(gdb); - } - - let run = |cmd: &mut Command| { - cmd.output().map(|output| { - String::from_utf8_lossy(&output.stdout) - .lines() - .next() - .unwrap_or_else(|| panic!("{:?} failed {:?}", cmd, output)) - .to_string() - }) - }; - let lldb_exe = "lldb"; - let lldb_version = Command::new(lldb_exe) - .arg("--version") - .output() - .map(|output| String::from_utf8_lossy(&output.stdout).to_string()) - .ok(); - if let Some(ref vers) = lldb_version { - cmd.arg("--lldb-version").arg(vers); - let lldb_python_dir = run(Command::new(lldb_exe).arg("-P")).ok(); - if let Some(ref dir) = lldb_python_dir { - cmd.arg("--lldb-python-dir").arg(dir); - } - } - - if util::forcing_clang_based_tests() { - let clang_exe = builder.llvm_out(target).join("bin").join("clang"); - cmd.arg("--run-clang-based-tests-with").arg(clang_exe); - } - - for exclude in &builder.config.skip { - cmd.arg("--skip"); - cmd.arg(&exclude); - } - - // Get paths from cmd args - let paths = match &builder.config.cmd { - Subcommand::Test { .. } => &builder.config.paths[..], - _ => &[], - }; - - // Get test-args by striping suite path - let mut test_args: Vec<&str> = paths - .iter() - .filter_map(|p| util::is_valid_test_suite_arg(p, suite_path, builder)) - .collect(); - - test_args.append(&mut builder.config.test_args()); - - // On Windows, replace forward slashes in test-args by backslashes - // so the correct filters are passed to libtest - if cfg!(windows) { - let test_args_win: Vec = - test_args.iter().map(|s| s.replace("/", "\\")).collect(); - cmd.args(&test_args_win); - } else { - cmd.args(&test_args); - } - - if builder.is_verbose() { - cmd.arg("--verbose"); - } - - cmd.arg("--json"); - - let mut llvm_components_passed = false; - let mut copts_passed = false; - if builder.config.llvm_enabled() { - let llvm::LlvmResult { llvm_config, .. } = - builder.ensure(llvm::Llvm { target: builder.config.build }); - if !builder.config.dry_run() { - let llvm_version = output(Command::new(&llvm_config).arg("--version")); - let llvm_components = output(Command::new(&llvm_config).arg("--components")); - // Remove trailing newline from llvm-config output. - cmd.arg("--llvm-version") - .arg(llvm_version.trim()) - .arg("--llvm-components") - .arg(llvm_components.trim()); - llvm_components_passed = true; - } - if !builder.is_rust_llvm(target) { - cmd.arg("--system-llvm"); - } - - // Tests that use compiler libraries may inherit the `-lLLVM` link - // requirement, but the `-L` library path is not propagated across - // separate compilations. We can add LLVM's library path to the - // platform-specific environment variable as a workaround. - if !builder.config.dry_run() && suite.ends_with("fulldeps") { - let llvm_libdir = output(Command::new(&llvm_config).arg("--libdir")); - add_link_lib_path(vec![llvm_libdir.trim().into()], &mut cmd); - } - - if !builder.config.dry_run() - && (matches!(suite, "run-make" | "run-make-fulldeps") || mode == "run-coverage") - { - // The llvm/bin directory contains many useful cross-platform - // tools. Pass the path to run-make tests so they can use them. - // (The run-coverage tests also need these tools to process - // coverage reports.) - let llvm_bin_path = llvm_config - .parent() - .expect("Expected llvm-config to be contained in directory"); - assert!(llvm_bin_path.is_dir()); - cmd.arg("--llvm-bin-dir").arg(llvm_bin_path); - } - - if !builder.config.dry_run() && matches!(suite, "run-make" | "run-make-fulldeps") { - // If LLD is available, add it to the PATH - if builder.config.lld_enabled { - let lld_install_root = - builder.ensure(llvm::Lld { target: builder.config.build }); - - let lld_bin_path = lld_install_root.join("bin"); - - let old_path = env::var_os("PATH").unwrap_or_default(); - let new_path = env::join_paths( - std::iter::once(lld_bin_path).chain(env::split_paths(&old_path)), - ) - .expect("Could not add LLD bin path to PATH"); - cmd.env("PATH", new_path); - } - } - } - - // Only pass correct values for these flags for the `run-make` suite as it - // requires that a C++ compiler was configured which isn't always the case. - if !builder.config.dry_run() && matches!(suite, "run-make" | "run-make-fulldeps") { - cmd.arg("--cc") - .arg(builder.cc(target)) - .arg("--cxx") - .arg(builder.cxx(target).unwrap()) - .arg("--cflags") - .arg(builder.cflags(target, GitRepo::Rustc, CLang::C).join(" ")) - .arg("--cxxflags") - .arg(builder.cflags(target, GitRepo::Rustc, CLang::Cxx).join(" ")); - copts_passed = true; - if let Some(ar) = builder.ar(target) { - cmd.arg("--ar").arg(ar); - } - } - - if !llvm_components_passed { - cmd.arg("--llvm-components").arg(""); - } - if !copts_passed { - cmd.arg("--cc") - .arg("") - .arg("--cxx") - .arg("") - .arg("--cflags") - .arg("") - .arg("--cxxflags") - .arg(""); - } - - if builder.remote_tested(target) { - cmd.arg("--remote-test-client").arg(builder.tool_exe(Tool::RemoteTestClient)); - } - - // Running a C compiler on MSVC requires a few env vars to be set, to be - // sure to set them here. - // - // Note that if we encounter `PATH` we make sure to append to our own `PATH` - // rather than stomp over it. - if !builder.config.dry_run() && target.contains("msvc") { - for &(ref k, ref v) in builder.cc.borrow()[&target].env() { - if k != "PATH" { - cmd.env(k, v); - } - } - } - cmd.env("RUSTC_BOOTSTRAP", "1"); - // Override the rustc version used in symbol hashes to reduce the amount of normalization - // needed when diffing test output. - cmd.env("RUSTC_FORCE_RUSTC_VERSION", "compiletest"); - cmd.env("DOC_RUST_LANG_ORG_CHANNEL", builder.doc_rust_lang_org_channel()); - builder.add_rust_test_threads(&mut cmd); - - if builder.config.sanitizers_enabled(target) { - cmd.env("RUSTC_SANITIZER_SUPPORT", "1"); - } - - if builder.config.profiler_enabled(target) { - cmd.env("RUSTC_PROFILER_SUPPORT", "1"); - } - - cmd.env("RUST_TEST_TMPDIR", builder.tempdir()); - - cmd.arg("--adb-path").arg("adb"); - cmd.arg("--adb-test-dir").arg(ADB_TEST_DIR); - if target.contains("android") && !builder.config.dry_run() { - // Assume that cc for this target comes from the android sysroot - cmd.arg("--android-cross-path") - .arg(builder.cc(target).parent().unwrap().parent().unwrap()); - } else { - cmd.arg("--android-cross-path").arg(""); - } - - if builder.config.cmd.rustfix_coverage() { - cmd.arg("--rustfix-coverage"); - } - - cmd.env("BOOTSTRAP_CARGO", &builder.initial_cargo); - - cmd.arg("--channel").arg(&builder.config.channel); - - if !builder.config.omit_git_hash { - cmd.arg("--git-hash"); - } - - builder.ci_env.force_coloring_in_ci(&mut cmd); - - #[cfg(feature = "build-metrics")] - builder.metrics.begin_test_suite( - build_helper::metrics::TestSuiteMetadata::Compiletest { - suite: suite.into(), - mode: mode.into(), - compare_mode: None, - target: self.target.triple.to_string(), - host: self.compiler.host.triple.to_string(), - stage: self.compiler.stage, - }, - builder, - ); - - let _group = builder.msg( - Kind::Test, - compiler.stage, - &format!("compiletest suite={suite} mode={mode}"), - compiler.host, - target, - ); - crate::render_tests::try_run_tests(builder, &mut cmd, false); - - if let Some(compare_mode) = compare_mode { - cmd.arg("--compare-mode").arg(compare_mode); - - #[cfg(feature = "build-metrics")] - builder.metrics.begin_test_suite( - build_helper::metrics::TestSuiteMetadata::Compiletest { - suite: suite.into(), - mode: mode.into(), - compare_mode: Some(compare_mode.into()), - target: self.target.triple.to_string(), - host: self.compiler.host.triple.to_string(), - stage: self.compiler.stage, - }, - builder, - ); - - builder.info(&format!( - "Check compiletest suite={} mode={} compare_mode={} ({} -> {})", - suite, mode, compare_mode, &compiler.host, target - )); - let _time = util::timeit(&builder); - crate::render_tests::try_run_tests(builder, &mut cmd, false); - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -struct BookTest { - compiler: Compiler, - path: PathBuf, - name: &'static str, - is_ext_doc: bool, -} - -impl Step for BookTest { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Runs the documentation tests for a book in `src/doc`. - /// - /// This uses the `rustdoc` that sits next to `compiler`. - fn run(self, builder: &Builder<'_>) { - // External docs are different from local because: - // - Some books need pre-processing by mdbook before being tested. - // - They need to save their state to toolstate. - // - They are only tested on the "checktools" builders. - // - // The local docs are tested by default, and we don't want to pay the - // cost of building mdbook, so they use `rustdoc --test` directly. - // Also, the unstable book is special because SUMMARY.md is generated, - // so it is easier to just run `rustdoc` on its files. - if self.is_ext_doc { - self.run_ext_doc(builder); - } else { - self.run_local_doc(builder); - } - } -} - -impl BookTest { - /// This runs the equivalent of `mdbook test` (via the rustbook wrapper) - /// which in turn runs `rustdoc --test` on each file in the book. - fn run_ext_doc(self, builder: &Builder<'_>) { - let compiler = self.compiler; - - builder.ensure(compile::Std::new(compiler, compiler.host)); - - // mdbook just executes a binary named "rustdoc", so we need to update - // PATH so that it points to our rustdoc. - let mut rustdoc_path = builder.rustdoc(compiler); - rustdoc_path.pop(); - let old_path = env::var_os("PATH").unwrap_or_default(); - let new_path = env::join_paths(iter::once(rustdoc_path).chain(env::split_paths(&old_path))) - .expect("could not add rustdoc to PATH"); - - let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook); - let path = builder.src.join(&self.path); - // Books often have feature-gated example text. - rustbook_cmd.env("RUSTC_BOOTSTRAP", "1"); - rustbook_cmd.env("PATH", new_path).arg("test").arg(path); - builder.add_rust_test_threads(&mut rustbook_cmd); - let _guard = builder.msg( - Kind::Test, - compiler.stage, - format_args!("mdbook {}", self.path.display()), - compiler.host, - compiler.host, - ); - let _time = util::timeit(&builder); - let toolstate = if builder.run_delaying_failure(&mut rustbook_cmd) { - ToolState::TestPass - } else { - ToolState::TestFail - }; - builder.save_toolstate(self.name, toolstate); - } - - /// This runs `rustdoc --test` on all `.md` files in the path. - fn run_local_doc(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let host = self.compiler.host; - - builder.ensure(compile::Std::new(compiler, host)); - - let _guard = - builder.msg(Kind::Test, compiler.stage, &format!("book {}", self.name), host, host); - - // Do a breadth-first traversal of the `src/doc` directory and just run - // tests for all files that end in `*.md` - let mut stack = vec![builder.src.join(self.path)]; - let _time = util::timeit(&builder); - let mut files = Vec::new(); - while let Some(p) = stack.pop() { - if p.is_dir() { - stack.extend(t!(p.read_dir()).map(|p| t!(p).path())); - continue; - } - - if p.extension().and_then(|s| s.to_str()) != Some("md") { - continue; - } - - files.push(p); - } - - files.sort(); - - for file in files { - markdown_test(builder, compiler, &file); - } - } -} - -macro_rules! test_book { - ($($name:ident, $path:expr, $book_name:expr, default=$default:expr;)+) => { - $( - #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] - pub struct $name { - compiler: Compiler, - } - - impl Step for $name { - type Output = (); - const DEFAULT: bool = $default; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path($path) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure($name { - compiler: run.builder.compiler(run.builder.top_stage, run.target), - }); - } - - fn run(self, builder: &Builder<'_>) { - builder.ensure(BookTest { - compiler: self.compiler, - path: PathBuf::from($path), - name: $book_name, - is_ext_doc: !$default, - }); - } - } - )+ - } -} - -test_book!( - Nomicon, "src/doc/nomicon", "nomicon", default=false; - Reference, "src/doc/reference", "reference", default=false; - RustdocBook, "src/doc/rustdoc", "rustdoc", default=true; - RustcBook, "src/doc/rustc", "rustc", default=true; - RustByExample, "src/doc/rust-by-example", "rust-by-example", default=false; - EmbeddedBook, "src/doc/embedded-book", "embedded-book", default=false; - TheBook, "src/doc/book", "book", default=false; - UnstableBook, "src/doc/unstable-book", "unstable-book", default=true; - EditionGuide, "src/doc/edition-guide", "edition-guide", default=false; -); - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct ErrorIndex { - compiler: Compiler, -} - -impl Step for ErrorIndex { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/error_index_generator") - } - - fn make_run(run: RunConfig<'_>) { - // error_index_generator depends on librustdoc. Use the compiler that - // is normally used to build rustdoc for other tests (like compiletest - // tests in tests/rustdoc) so that it shares the same artifacts. - let compiler = run.builder.compiler(run.builder.top_stage, run.builder.config.build); - run.builder.ensure(ErrorIndex { compiler }); - } - - /// Runs the error index generator tool to execute the tests located in the error - /// index. - /// - /// The `error_index_generator` tool lives in `src/tools` and is used to - /// generate a markdown file from the error indexes of the code base which is - /// then passed to `rustdoc --test`. - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - - let dir = testdir(builder, compiler.host); - t!(fs::create_dir_all(&dir)); - let output = dir.join("error-index.md"); - - let mut tool = tool::ErrorIndex::command(builder); - tool.arg("markdown").arg(&output); - - let guard = - builder.msg(Kind::Test, compiler.stage, "error-index", compiler.host, compiler.host); - let _time = util::timeit(&builder); - builder.run_quiet(&mut tool); - drop(guard); - // The tests themselves need to link to std, so make sure it is - // available. - builder.ensure(compile::Std::new(compiler, compiler.host)); - markdown_test(builder, compiler, &output); - } -} - -fn markdown_test(builder: &Builder<'_>, compiler: Compiler, markdown: &Path) -> bool { - if let Ok(contents) = fs::read_to_string(markdown) { - if !contents.contains("```") { - return true; - } - } - - builder.verbose(&format!("doc tests for: {}", markdown.display())); - let mut cmd = builder.rustdoc_cmd(compiler); - builder.add_rust_test_threads(&mut cmd); - // allow for unstable options such as new editions - cmd.arg("-Z"); - cmd.arg("unstable-options"); - cmd.arg("--test"); - cmd.arg(markdown); - cmd.env("RUSTC_BOOTSTRAP", "1"); - - let test_args = builder.config.test_args().join(" "); - cmd.arg("--test-args").arg(test_args); - - if builder.config.verbose_tests { - builder.run_delaying_failure(&mut cmd) - } else { - builder.run_quiet_delaying_failure(&mut cmd) - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct RustcGuide; - -impl Step for RustcGuide { - type Output = (); - const DEFAULT: bool = false; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/doc/rustc-dev-guide") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustcGuide); - } - - fn run(self, builder: &Builder<'_>) { - let relative_path = Path::new("src").join("doc").join("rustc-dev-guide"); - builder.update_submodule(&relative_path); - - let src = builder.src.join(relative_path); - let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook); - let toolstate = if builder.run_delaying_failure(rustbook_cmd.arg("linkcheck").arg(&src)) { - ToolState::TestPass - } else { - ToolState::TestFail - }; - builder.save_toolstate("rustc-dev-guide", toolstate); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct CrateLibrustc { - compiler: Compiler, - target: TargetSelection, - crates: Vec>, -} - -impl Step for CrateLibrustc { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.crate_or_deps("rustc-main") - } - - fn make_run(run: RunConfig<'_>) { - let builder = run.builder; - let host = run.build_triple(); - let compiler = builder.compiler_for(builder.top_stage, host, host); - let crates = run - .paths - .iter() - .map(|p| builder.crate_paths[&p.assert_single_path().path].clone()) - .collect(); - - builder.ensure(CrateLibrustc { compiler, target: run.target, crates }); - } - - fn run(self, builder: &Builder<'_>) { - builder.ensure(Crate { - compiler: self.compiler, - target: self.target, - mode: Mode::Rustc, - crates: self.crates, - }); - } -} - -/// Given a `cargo test` subcommand, add the appropriate flags and run it. -/// -/// Returns whether the test succeeded. -fn run_cargo_test<'a>( - cargo: impl Into, - libtest_args: &[&str], - crates: &[Interned], - primary_crate: &str, - description: impl Into>, - compiler: Compiler, - target: TargetSelection, - builder: &Builder<'_>, -) -> bool { - let mut cargo = - prepare_cargo_test(cargo, libtest_args, crates, primary_crate, compiler, target, builder); - let _time = util::timeit(&builder); - let _group = description.into().and_then(|what| { - builder.msg_sysroot_tool(Kind::Test, compiler.stage, what, compiler.host, target) - }); - - #[cfg(feature = "build-metrics")] - builder.metrics.begin_test_suite( - build_helper::metrics::TestSuiteMetadata::CargoPackage { - crates: crates.iter().map(|c| c.to_string()).collect(), - target: target.triple.to_string(), - host: compiler.host.triple.to_string(), - stage: compiler.stage, - }, - builder, - ); - add_flags_and_try_run_tests(builder, &mut cargo) -} - -/// Given a `cargo test` subcommand, pass it the appropriate test flags given a `builder`. -fn prepare_cargo_test( - cargo: impl Into, - libtest_args: &[&str], - crates: &[Interned], - primary_crate: &str, - compiler: Compiler, - target: TargetSelection, - builder: &Builder<'_>, -) -> Command { - let mut cargo = cargo.into(); - - // Propegate `--bless` if it has not already been set/unset - // Any tools that want to use this should bless if `RUSTC_BLESS` is set to - // anything other than `0`. - if builder.config.cmd.bless() && !cargo.get_envs().any(|v| v.0 == "RUSTC_BLESS") { - cargo.env("RUSTC_BLESS", "Gesundheit"); - } - - // Pass in some standard flags then iterate over the graph we've discovered - // in `cargo metadata` with the maps above and figure out what `-p` - // arguments need to get passed. - if builder.kind == Kind::Test && !builder.fail_fast { - cargo.arg("--no-fail-fast"); - } - match builder.doc_tests { - DocTests::Only => { - cargo.arg("--doc"); - } - DocTests::No => { - let krate = &builder - .crates - .get(&INTERNER.intern_str(primary_crate)) - .unwrap_or_else(|| panic!("missing crate {primary_crate}")); - if krate.has_lib { - cargo.arg("--lib"); - } - cargo.args(&["--bins", "--examples", "--tests", "--benches"]); - } - DocTests::Yes => {} - } - - for &krate in crates { - cargo.arg("-p").arg(krate); - } - - cargo.arg("--").args(&builder.config.test_args()).args(libtest_args); - if !builder.config.verbose_tests { - cargo.arg("--quiet"); - } - - // The tests are going to run with the *target* libraries, so we need to - // ensure that those libraries show up in the LD_LIBRARY_PATH equivalent. - // - // Note that to run the compiler we need to run with the *host* libraries, - // but our wrapper scripts arrange for that to be the case anyway. - let mut dylib_path = dylib_path(); - dylib_path.insert(0, PathBuf::from(&*builder.sysroot_libdir(compiler, target))); - cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); - - if target.contains("emscripten") { - cargo.env( - format!("CARGO_TARGET_{}_RUNNER", envify(&target.triple)), - builder.config.nodejs.as_ref().expect("nodejs not configured"), - ); - } else if target.starts_with("wasm32") { - let node = builder.config.nodejs.as_ref().expect("nodejs not configured"); - let runner = format!("{} {}/src/etc/wasm32-shim.js", node.display(), builder.src.display()); - cargo.env(format!("CARGO_TARGET_{}_RUNNER", envify(&target.triple)), &runner); - } else if builder.remote_tested(target) { - cargo.env( - format!("CARGO_TARGET_{}_RUNNER", envify(&target.triple)), - format!("{} run 0", builder.tool_exe(Tool::RemoteTestClient).display()), - ); - } - - cargo -} - -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct Crate { - pub compiler: Compiler, - pub target: TargetSelection, - pub mode: Mode, - pub crates: Vec>, -} - -impl Step for Crate { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.crate_or_deps("sysroot") - } - - fn make_run(run: RunConfig<'_>) { - let builder = run.builder; - let host = run.build_triple(); - let compiler = builder.compiler_for(builder.top_stage, host, host); - let crates = run - .paths - .iter() - .map(|p| builder.crate_paths[&p.assert_single_path().path].clone()) - .collect(); - - builder.ensure(Crate { compiler, target: run.target, mode: Mode::Std, crates }); - } - - /// Runs all unit tests plus documentation tests for a given crate defined - /// by a `Cargo.toml` (single manifest) - /// - /// This is what runs tests for crates like the standard library, compiler, etc. - /// It essentially is the driver for running `cargo test`. - /// - /// Currently this runs all tests for a DAG by passing a bunch of `-p foo` - /// arguments, and those arguments are discovered from `cargo metadata`. - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let target = self.target; - let mode = self.mode; - - // See [field@compile::Std::force_recompile]. - builder.ensure(compile::Std::force_recompile(compiler, target)); - builder.ensure(RemoteCopyLibs { compiler, target }); - - // If we're not doing a full bootstrap but we're testing a stage2 - // version of libstd, then what we're actually testing is the libstd - // produced in stage1. Reflect that here by updating the compiler that - // we're working with automatically. - let compiler = builder.compiler_for(compiler.stage, compiler.host, target); - - let mut cargo = - builder.cargo(compiler, mode, SourceType::InTree, target, builder.kind.as_str()); - match mode { - Mode::Std => { - compile::std_cargo(builder, target, compiler.stage, &mut cargo); - // `std_cargo` actually does the wrong thing: it passes `--sysroot build/host/stage2`, - // but we want to use the force-recompile std we just built in `build/host/stage2-test-sysroot`. - // Override it. - if builder.download_rustc() && compiler.stage > 0 { - let sysroot = builder - .out - .join(compiler.host.triple) - .join(format!("stage{}-test-sysroot", compiler.stage)); - cargo.env("RUSTC_SYSROOT", sysroot); - } - } - Mode::Rustc => { - compile::rustc_cargo(builder, &mut cargo, target, compiler.stage); - } - _ => panic!("can only test libraries"), - }; - - run_cargo_test( - cargo, - &[], - &self.crates, - &self.crates[0], - &*crate_description(&self.crates), - compiler, - target, - builder, - ); - } -} - -/// Rustdoc is special in various ways, which is why this step is different from `Crate`. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct CrateRustdoc { - host: TargetSelection, -} - -impl Step for CrateRustdoc { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.paths(&["src/librustdoc", "src/tools/rustdoc"]) - } - - fn make_run(run: RunConfig<'_>) { - let builder = run.builder; - - builder.ensure(CrateRustdoc { host: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let target = self.host; - - let compiler = if builder.download_rustc() { - builder.compiler(builder.top_stage, target) - } else { - // Use the previous stage compiler to reuse the artifacts that are - // created when running compiletest for tests/rustdoc. If this used - // `compiler`, then it would cause rustdoc to be built *again*, which - // isn't really necessary. - builder.compiler_for(builder.top_stage, target, target) - }; - // NOTE: normally `ensure(Rustc)` automatically runs `ensure(Std)` for us. However, when - // using `download-rustc`, the rustc_private artifacts may be in a *different sysroot* from - // the target rustdoc (`ci-rustc-sysroot` vs `stage2`). In that case, we need to ensure this - // explicitly to make sure it ends up in the stage2 sysroot. - builder.ensure(compile::Std::new(compiler, target)); - builder.ensure(compile::Rustc::new(compiler, target)); - - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - target, - builder.kind.as_str(), - "src/tools/rustdoc", - SourceType::InTree, - &[], - ); - if self.host.contains("musl") { - cargo.arg("'-Ctarget-feature=-crt-static'"); - } - - // This is needed for running doctests on librustdoc. This is a bit of - // an unfortunate interaction with how bootstrap works and how cargo - // sets up the dylib path, and the fact that the doctest (in - // html/markdown.rs) links to rustc-private libs. For stage1, the - // compiler host dylibs (in stage1/lib) are not the same as the target - // dylibs (in stage1/lib/rustlib/...). This is different from a normal - // rust distribution where they are the same. - // - // On the cargo side, normal tests use `target_process` which handles - // setting up the dylib for a *target* (stage1/lib/rustlib/... in this - // case). However, for doctests it uses `rustdoc_process` which only - // sets up the dylib path for the *host* (stage1/lib), which is the - // wrong directory. - // - // Recall that we special-cased `compiler_for(top_stage)` above, so we always use stage1. - // - // It should be considered to just stop running doctests on - // librustdoc. There is only one test, and it doesn't look too - // important. There might be other ways to avoid this, but it seems - // pretty convoluted. - // - // See also https://github.com/rust-lang/rust/issues/13983 where the - // host vs target dylibs for rustdoc are consistently tricky to deal - // with. - // - // Note that this set the host libdir for `download_rustc`, which uses a normal rust distribution. - let libdir = if builder.download_rustc() { - builder.rustc_libdir(compiler) - } else { - builder.sysroot_libdir(compiler, target).to_path_buf() - }; - let mut dylib_path = dylib_path(); - dylib_path.insert(0, PathBuf::from(&*libdir)); - cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); - - run_cargo_test( - cargo, - &[], - &[INTERNER.intern_str("rustdoc:0.0.0")], - "rustdoc", - "rustdoc", - compiler, - target, - builder, - ); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct CrateRustdocJsonTypes { - host: TargetSelection, -} - -impl Step for CrateRustdocJsonTypes { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/rustdoc-json-types") - } - - fn make_run(run: RunConfig<'_>) { - let builder = run.builder; - - builder.ensure(CrateRustdocJsonTypes { host: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let target = self.host; - - // Use the previous stage compiler to reuse the artifacts that are - // created when running compiletest for tests/rustdoc. If this used - // `compiler`, then it would cause rustdoc to be built *again*, which - // isn't really necessary. - let compiler = builder.compiler_for(builder.top_stage, target, target); - builder.ensure(compile::Rustc::new(compiler, target)); - - let cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - target, - builder.kind.as_str(), - "src/rustdoc-json-types", - SourceType::InTree, - &[], - ); - - // FIXME: this looks very wrong, libtest doesn't accept `-C` arguments and the quotes are fishy. - let libtest_args = if self.host.contains("musl") { - ["'-Ctarget-feature=-crt-static'"].as_slice() - } else { - &[] - }; - - run_cargo_test( - cargo, - libtest_args, - &[INTERNER.intern_str("rustdoc-json-types")], - "rustdoc-json-types", - "rustdoc-json-types", - compiler, - target, - builder, - ); - } -} - -/// Some test suites are run inside emulators or on remote devices, and most -/// of our test binaries are linked dynamically which means we need to ship -/// the standard library and such to the emulator ahead of time. This step -/// represents this and is a dependency of all test suites. -/// -/// Most of the time this is a no-op. For some steps such as shipping data to -/// QEMU we have to build our own tools so we've got conditional dependencies -/// on those programs as well. Note that the remote test client is built for -/// the build target (us) and the server is built for the target. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct RemoteCopyLibs { - compiler: Compiler, - target: TargetSelection, -} - -impl Step for RemoteCopyLibs { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let target = self.target; - if !builder.remote_tested(target) { - return; - } - - builder.ensure(compile::Std::new(compiler, target)); - - builder.info(&format!("REMOTE copy libs to emulator ({target})")); - - let server = builder.ensure(tool::RemoteTestServer { compiler, target }); - - // Spawn the emulator and wait for it to come online - let tool = builder.tool_exe(Tool::RemoteTestClient); - let mut cmd = Command::new(&tool); - cmd.arg("spawn-emulator").arg(target.triple).arg(&server).arg(builder.tempdir()); - if let Some(rootfs) = builder.qemu_rootfs(target) { - cmd.arg(rootfs); - } - builder.run(&mut cmd); - - // Push all our dylibs to the emulator - for f in t!(builder.sysroot_libdir(compiler, target).read_dir()) { - let f = t!(f); - let name = f.file_name().into_string().unwrap(); - if util::is_dylib(&name) { - builder.run(Command::new(&tool).arg("push").arg(f.path())); - } - } - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Distcheck; - -impl Step for Distcheck { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("distcheck") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Distcheck); - } - - /// Runs "distcheck", a 'make check' from a tarball - fn run(self, builder: &Builder<'_>) { - builder.info("Distcheck"); - let dir = builder.tempdir().join("distcheck"); - let _ = fs::remove_dir_all(&dir); - t!(fs::create_dir_all(&dir)); - - // Guarantee that these are built before we begin running. - builder.ensure(dist::PlainSourceTarball); - builder.ensure(dist::Src); - - let mut cmd = Command::new("tar"); - cmd.arg("-xf") - .arg(builder.ensure(dist::PlainSourceTarball).tarball()) - .arg("--strip-components=1") - .current_dir(&dir); - builder.run(&mut cmd); - builder.run( - Command::new("./configure") - .args(&builder.config.configure_args) - .arg("--enable-vendor") - .current_dir(&dir), - ); - builder.run( - Command::new(util::make(&builder.config.build.triple)).arg("check").current_dir(&dir), - ); - - // Now make sure that rust-src has all of libstd's dependencies - builder.info("Distcheck rust-src"); - let dir = builder.tempdir().join("distcheck-src"); - let _ = fs::remove_dir_all(&dir); - t!(fs::create_dir_all(&dir)); - - let mut cmd = Command::new("tar"); - cmd.arg("-xf") - .arg(builder.ensure(dist::Src).tarball()) - .arg("--strip-components=1") - .current_dir(&dir); - builder.run(&mut cmd); - - let toml = dir.join("rust-src/lib/rustlib/src/rust/library/std/Cargo.toml"); - builder.run( - Command::new(&builder.initial_cargo) - // Will read the libstd Cargo.toml - // which uses the unstable `public-dependency` feature. - .env("RUSTC_BOOTSTRAP", "1") - .arg("generate-lockfile") - .arg("--manifest-path") - .arg(&toml) - .current_dir(&dir), - ); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Bootstrap; - -impl Step for Bootstrap { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - /// Tests the build system itself. - fn run(self, builder: &Builder<'_>) { - let host = builder.config.build; - let compiler = builder.compiler(0, host); - let _guard = builder.msg(Kind::Test, 0, "bootstrap", host, host); - - let mut check_bootstrap = Command::new(&builder.python()); - check_bootstrap - .args(["-m", "unittest", "bootstrap_test.py"]) - .env("BUILD_DIR", &builder.out) - .env("BUILD_PLATFORM", &builder.build.build.triple) - .current_dir(builder.src.join("src/bootstrap/")); - // NOTE: we intentionally don't pass test_args here because the args for unittest and cargo test are mutually incompatible. - // Use `python -m unittest` manually if you want to pass arguments. - builder.run_delaying_failure(&mut check_bootstrap); - - let mut cmd = Command::new(&builder.initial_cargo); - cmd.arg("test") - .current_dir(builder.src.join("src/bootstrap")) - .env("RUSTFLAGS", "-Cdebuginfo=2") - .env("CARGO_TARGET_DIR", builder.out.join("bootstrap")) - .env("RUSTC_BOOTSTRAP", "1") - .env("RUSTDOC", builder.rustdoc(compiler)) - .env("RUSTC", &builder.initial_rustc); - if let Some(flags) = option_env!("RUSTFLAGS") { - // Use the same rustc flags for testing as for "normal" compilation, - // so that Cargo doesn’t recompile the entire dependency graph every time: - // https://github.com/rust-lang/rust/issues/49215 - cmd.env("RUSTFLAGS", flags); - } - // rustbuild tests are racy on directory creation so just run them one at a time. - // Since there's not many this shouldn't be a problem. - run_cargo_test(cmd, &["--test-threads=1"], &[], "bootstrap", None, compiler, host, builder); - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/bootstrap") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Bootstrap); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct TierCheck { - pub compiler: Compiler, -} - -impl Step for TierCheck { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/tier-check") - } - - fn make_run(run: RunConfig<'_>) { - let compiler = - run.builder.compiler_for(run.builder.top_stage, run.builder.build.build, run.target); - run.builder.ensure(TierCheck { compiler }); - } - - /// Tests the Platform Support page in the rustc book. - fn run(self, builder: &Builder<'_>) { - builder.ensure(compile::Std::new(self.compiler, self.compiler.host)); - let mut cargo = tool::prepare_tool_cargo( - builder, - self.compiler, - Mode::ToolStd, - self.compiler.host, - "run", - "src/tools/tier-check", - SourceType::InTree, - &[], - ); - cargo.arg(builder.src.join("src/doc/rustc/src/platform-support.md")); - cargo.arg(&builder.rustc(self.compiler)); - if builder.is_verbose() { - cargo.arg("--verbose"); - } - - let _guard = builder.msg( - Kind::Test, - self.compiler.stage, - "platform support check", - self.compiler.host, - self.compiler.host, - ); - builder.run_delaying_failure(&mut cargo.into()); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct LintDocs { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for LintDocs { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/lint-docs") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(LintDocs { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - /// Tests that the lint examples in the rustc book generate the correct - /// lints and have the expected format. - fn run(self, builder: &Builder<'_>) { - builder.ensure(crate::doc::RustcBook { - compiler: self.compiler, - target: self.target, - validate: true, - }); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct RustInstaller; - -impl Step for RustInstaller { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - /// Ensure the version placeholder replacement tool builds - fn run(self, builder: &Builder<'_>) { - let bootstrap_host = builder.config.build; - let compiler = builder.compiler(0, bootstrap_host); - let cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolBootstrap, - bootstrap_host, - "test", - "src/tools/rust-installer", - SourceType::InTree, - &[], - ); - - let _guard = builder.msg( - Kind::Test, - compiler.stage, - "rust-installer", - bootstrap_host, - bootstrap_host, - ); - run_cargo_test(cargo, &[], &[], "installer", None, compiler, bootstrap_host, builder); - - // We currently don't support running the test.sh script outside linux(?) environments. - // Eventually this should likely migrate to #[test]s in rust-installer proper rather than a - // set of scripts, which will likely allow dropping this if. - if bootstrap_host != "x86_64-unknown-linux-gnu" { - return; - } - - let mut cmd = - std::process::Command::new(builder.src.join("src/tools/rust-installer/test.sh")); - let tmpdir = testdir(builder, compiler.host).join("rust-installer"); - let _ = std::fs::remove_dir_all(&tmpdir); - let _ = std::fs::create_dir_all(&tmpdir); - cmd.current_dir(&tmpdir); - cmd.env("CARGO_TARGET_DIR", tmpdir.join("cargo-target")); - cmd.env("CARGO", &builder.initial_cargo); - cmd.env("RUSTC", &builder.initial_rustc); - cmd.env("TMP_DIR", &tmpdir); - builder.run_delaying_failure(&mut cmd); - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/rust-installer") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Self); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct TestHelpers { - pub target: TargetSelection, -} - -impl Step for TestHelpers { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("tests/auxiliary/rust_test_helpers.c") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(TestHelpers { target: run.target }) - } - - /// Compiles the `rust_test_helpers.c` library which we used in various - /// `run-pass` tests for ABI testing. - fn run(self, builder: &Builder<'_>) { - if builder.config.dry_run() { - return; - } - // The x86_64-fortanix-unknown-sgx target doesn't have a working C - // toolchain. However, some x86_64 ELF objects can be linked - // without issues. Use this hack to compile the test helpers. - let target = if self.target == "x86_64-fortanix-unknown-sgx" { - TargetSelection::from_user("x86_64-unknown-linux-gnu") - } else { - self.target - }; - let dst = builder.test_helpers_out(target); - let src = builder.src.join("tests/auxiliary/rust_test_helpers.c"); - if up_to_date(&src, &dst.join("librust_test_helpers.a")) { - return; - } - - let _guard = builder.msg_unstaged(Kind::Build, "test helpers", target); - t!(fs::create_dir_all(&dst)); - let mut cfg = cc::Build::new(); - // FIXME: Workaround for https://github.com/emscripten-core/emscripten/issues/9013 - if target.contains("emscripten") { - cfg.pic(false); - } - - // We may have found various cross-compilers a little differently due to our - // extra configuration, so inform cc of these compilers. Note, though, that - // on MSVC we still need cc's detection of env vars (ugh). - if !target.contains("msvc") { - if let Some(ar) = builder.ar(target) { - cfg.archiver(ar); - } - cfg.compiler(builder.cc(target)); - } - cfg.cargo_metadata(false) - .out_dir(&dst) - .target(&target.triple) - .host(&builder.config.build.triple) - .opt_level(0) - .warnings(false) - .debug(false) - .file(builder.src.join("tests/auxiliary/rust_test_helpers.c")) - .compile("rust_test_helpers"); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct CodegenCranelift { - compiler: Compiler, - target: TargetSelection, -} - -impl Step for CodegenCranelift { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.paths(&["compiler/rustc_codegen_cranelift"]) - } - - fn make_run(run: RunConfig<'_>) { - let builder = run.builder; - let host = run.build_triple(); - let compiler = run.builder.compiler_for(run.builder.top_stage, host, host); - - if builder.doc_tests == DocTests::Only { - return; - } - - let triple = run.target.triple; - let target_supported = if triple.contains("linux") { - triple.contains("x86_64") || triple.contains("aarch64") || triple.contains("s390x") - } else if triple.contains("darwin") || triple.contains("windows") { - triple.contains("x86_64") - } else { - false - }; - if !target_supported { - builder.info("target not supported by rustc_codegen_cranelift. skipping"); - return; - } - - if builder.remote_tested(run.target) { - builder.info("remote testing is not supported by rustc_codegen_cranelift. skipping"); - return; - } - - if !builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("cranelift")) { - builder.info("cranelift not in rust.codegen-backends. skipping"); - return; - } - - builder.ensure(CodegenCranelift { compiler, target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let target = self.target; - - builder.ensure(compile::Std::new(compiler, target)); - - // If we're not doing a full bootstrap but we're testing a stage2 - // version of libstd, then what we're actually testing is the libstd - // produced in stage1. Reflect that here by updating the compiler that - // we're working with automatically. - let compiler = builder.compiler_for(compiler.stage, compiler.host, target); - - let build_cargo = || { - let mut cargo = builder.cargo( - compiler, - Mode::Codegen, // Must be codegen to ensure dlopen on compiled dylibs works - SourceType::InTree, - target, - "run", - ); - cargo.current_dir(&builder.src.join("compiler/rustc_codegen_cranelift")); - cargo - .arg("--manifest-path") - .arg(builder.src.join("compiler/rustc_codegen_cranelift/build_system/Cargo.toml")); - compile::rustc_cargo_env(builder, &mut cargo, target, compiler.stage); - - // Avoid incremental cache issues when changing rustc - cargo.env("CARGO_BUILD_INCREMENTAL", "false"); - - cargo - }; - - builder.info(&format!( - "{} cranelift stage{} ({} -> {})", - Kind::Test.description(), - compiler.stage, - &compiler.host, - target - )); - let _time = util::timeit(&builder); - - // FIXME handle vendoring for source tarballs before removing the --skip-test below - let download_dir = builder.out.join("cg_clif_download"); - - /* - let mut prepare_cargo = build_cargo(); - prepare_cargo.arg("--").arg("prepare").arg("--download-dir").arg(&download_dir); - #[allow(deprecated)] - builder.config.try_run(&mut prepare_cargo.into()).unwrap(); - */ - - let mut cargo = build_cargo(); - cargo - .arg("--") - .arg("test") - .arg("--download-dir") - .arg(&download_dir) - .arg("--out-dir") - .arg(builder.stage_out(compiler, Mode::ToolRustc).join("cg_clif")) - .arg("--no-unstable-features") - .arg("--use-backend") - .arg("cranelift") - // Avoid having to vendor the standard library dependencies - .arg("--sysroot") - .arg("llvm") - // These tests depend on crates that are not yet vendored - // FIXME remove once vendoring is handled - .arg("--skip-test") - .arg("testsuite.extended_sysroot"); - cargo.args(builder.config.test_args()); - - #[allow(deprecated)] - builder.config.try_run(&mut cargo.into()).unwrap(); - } -} diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs deleted file mode 100644 index f094dd9d7..000000000 --- a/src/bootstrap/tool.rs +++ /dev/null @@ -1,848 +0,0 @@ -use std::env; -use std::fs; -use std::path::PathBuf; -use std::process::Command; - -use crate::builder::{Builder, Cargo as CargoCommand, RunConfig, ShouldRun, Step}; -use crate::channel::GitInfo; -use crate::compile; -use crate::config::TargetSelection; -use crate::toolstate::ToolState; -use crate::util::{add_dylib_path, exe, t}; -use crate::Compiler; -use crate::Mode; -use crate::{gha, Kind}; - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub enum SourceType { - InTree, - Submodule, -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -struct ToolBuild { - compiler: Compiler, - target: TargetSelection, - tool: &'static str, - path: &'static str, - mode: Mode, - is_optional_tool: bool, - source_type: SourceType, - extra_features: Vec, - /// Nightly-only features that are allowed (comma-separated list). - allow_features: &'static str, -} - -impl Builder<'_> { - #[track_caller] - fn msg_tool( - &self, - mode: Mode, - tool: &str, - build_stage: u32, - host: &TargetSelection, - target: &TargetSelection, - ) -> Option { - match mode { - // depends on compiler stage, different to host compiler - Mode::ToolRustc => self.msg_sysroot_tool( - Kind::Build, - build_stage, - format_args!("tool {tool}"), - *host, - *target, - ), - // doesn't depend on compiler, same as host compiler - _ => self.msg(Kind::Build, build_stage, format_args!("tool {tool}"), *host, *target), - } - } -} - -impl Step for ToolBuild { - type Output = Option; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Builds a tool in `src/tools` - /// - /// This will build the specified tool with the specified `host` compiler in - /// `stage` into the normal cargo output directory. - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - let mut tool = self.tool; - let path = self.path; - let is_optional_tool = self.is_optional_tool; - - match self.mode { - Mode::ToolRustc => { - builder.ensure(compile::Std::new(compiler, compiler.host)); - builder.ensure(compile::Rustc::new(compiler, target)); - } - Mode::ToolStd => builder.ensure(compile::Std::new(compiler, target)), - Mode::ToolBootstrap => {} // uses downloaded stage0 compiler libs - _ => panic!("unexpected Mode for tool build"), - } - - let mut cargo = prepare_tool_cargo( - builder, - compiler, - self.mode, - target, - "build", - path, - self.source_type, - &self.extra_features, - ); - if !self.allow_features.is_empty() { - cargo.allow_features(self.allow_features); - } - let _guard = builder.msg_tool( - self.mode, - self.tool, - self.compiler.stage, - &self.compiler.host, - &self.target, - ); - - let mut cargo = Command::from(cargo); - #[allow(deprecated)] // we check this in `is_optional_tool` in a second - let is_expected = builder.config.try_run(&mut cargo).is_ok(); - - builder.save_toolstate( - tool, - if is_expected { ToolState::TestFail } else { ToolState::BuildFail }, - ); - - if !is_expected { - if !is_optional_tool { - crate::exit!(1); - } else { - None - } - } else { - // HACK(#82501): on Windows, the tools directory gets added to PATH when running tests, and - // compiletest confuses HTML tidy with the in-tree tidy. Name the in-tree tidy something - // different so the problem doesn't come up. - if tool == "tidy" { - tool = "rust-tidy"; - } - let cargo_out = builder.cargo_out(compiler, self.mode, target).join(exe(tool, target)); - let bin = builder.tools_dir(compiler).join(exe(tool, target)); - builder.copy(&cargo_out, &bin); - Some(bin) - } - } -} - -pub fn prepare_tool_cargo( - builder: &Builder<'_>, - compiler: Compiler, - mode: Mode, - target: TargetSelection, - command: &'static str, - path: &str, - source_type: SourceType, - extra_features: &[String], -) -> CargoCommand { - let mut cargo = builder.cargo(compiler, mode, source_type, target, command); - let dir = builder.src.join(path); - cargo.arg("--manifest-path").arg(dir.join("Cargo.toml")); - - let mut features = extra_features.to_vec(); - if builder.build.config.cargo_native_static { - if path.ends_with("cargo") - || path.ends_with("rls") - || path.ends_with("clippy") - || path.ends_with("miri") - || path.ends_with("rustfmt") - { - cargo.env("LIBZ_SYS_STATIC", "1"); - } - if path.ends_with("cargo") { - features.push("all-static".to_string()); - } - } - - // clippy tests need to know about the stage sysroot. Set them consistently while building to - // avoid rebuilding when running tests. - cargo.env("SYSROOT", builder.sysroot(compiler)); - - // if tools are using lzma we want to force the build script to build its - // own copy - cargo.env("LZMA_API_STATIC", "1"); - - // CFG_RELEASE is needed by rustfmt (and possibly other tools) which - // import rustc-ap-rustc_attr which requires this to be set for the - // `#[cfg(version(...))]` attribute. - cargo.env("CFG_RELEASE", builder.rust_release()); - cargo.env("CFG_RELEASE_CHANNEL", &builder.config.channel); - cargo.env("CFG_VERSION", builder.rust_version()); - cargo.env("CFG_RELEASE_NUM", &builder.version); - cargo.env("DOC_RUST_LANG_ORG_CHANNEL", builder.doc_rust_lang_org_channel()); - if let Some(ref ver_date) = builder.rust_info().commit_date() { - cargo.env("CFG_VER_DATE", ver_date); - } - if let Some(ref ver_hash) = builder.rust_info().sha() { - cargo.env("CFG_VER_HASH", ver_hash); - } - - let info = GitInfo::new(builder.config.omit_git_hash, &dir); - if let Some(sha) = info.sha() { - cargo.env("CFG_COMMIT_HASH", sha); - } - if let Some(sha_short) = info.sha_short() { - cargo.env("CFG_SHORT_COMMIT_HASH", sha_short); - } - if let Some(date) = info.commit_date() { - cargo.env("CFG_COMMIT_DATE", date); - } - if !features.is_empty() { - cargo.arg("--features").arg(&features.join(", ")); - } - cargo -} - -macro_rules! bootstrap_tool { - ($( - $name:ident, $path:expr, $tool_name:expr - $(,is_external_tool = $external:expr)* - $(,is_unstable_tool = $unstable:expr)* - $(,allow_features = $allow_features:expr)? - ; - )+) => { - #[derive(Copy, PartialEq, Eq, Clone)] - pub enum Tool { - $( - $name, - )+ - } - - impl<'a> Builder<'a> { - pub fn tool_exe(&self, tool: Tool) -> PathBuf { - match tool { - $(Tool::$name => - self.ensure($name { - compiler: self.compiler(0, self.config.build), - target: self.config.build, - }), - )+ - } - } - } - - $( - #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] - pub struct $name { - pub compiler: Compiler, - pub target: TargetSelection, - } - - impl Step for $name { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path($path) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure($name { - // snapshot compiler - compiler: run.builder.compiler(0, run.builder.config.build), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: $tool_name, - mode: if false $(|| $unstable)* { - // use in-tree libraries for unstable features - Mode::ToolStd - } else { - Mode::ToolBootstrap - }, - path: $path, - is_optional_tool: false, - source_type: if false $(|| $external)* { - SourceType::Submodule - } else { - SourceType::InTree - }, - extra_features: vec![], - allow_features: concat!($($allow_features)*), - }).expect("expected to build -- essential tool") - } - } - )+ - } -} - -bootstrap_tool!( - Rustbook, "src/tools/rustbook", "rustbook"; - UnstableBookGen, "src/tools/unstable-book-gen", "unstable-book-gen"; - Tidy, "src/tools/tidy", "tidy"; - Linkchecker, "src/tools/linkchecker", "linkchecker"; - CargoTest, "src/tools/cargotest", "cargotest"; - Compiletest, "src/tools/compiletest", "compiletest", is_unstable_tool = true, allow_features = "test"; - BuildManifest, "src/tools/build-manifest", "build-manifest"; - RemoteTestClient, "src/tools/remote-test-client", "remote-test-client"; - RustInstaller, "src/tools/rust-installer", "rust-installer"; - RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes"; - ExpandYamlAnchors, "src/tools/expand-yaml-anchors", "expand-yaml-anchors"; - LintDocs, "src/tools/lint-docs", "lint-docs"; - JsonDocCk, "src/tools/jsondocck", "jsondocck"; - JsonDocLint, "src/tools/jsondoclint", "jsondoclint"; - HtmlChecker, "src/tools/html-checker", "html-checker"; - BumpStage0, "src/tools/bump-stage0", "bump-stage0"; - ReplaceVersionPlaceholder, "src/tools/replace-version-placeholder", "replace-version-placeholder"; - CollectLicenseMetadata, "src/tools/collect-license-metadata", "collect-license-metadata"; - GenerateCopyright, "src/tools/generate-copyright", "generate-copyright"; - SuggestTests, "src/tools/suggest-tests", "suggest-tests"; - GenerateWindowsSys, "src/tools/generate-windows-sys", "generate-windows-sys"; - RustdocGUITest, "src/tools/rustdoc-gui-test", "rustdoc-gui-test", is_unstable_tool = true, allow_features = "test"; - OptimizedDist, "src/tools/opt-dist", "opt-dist"; - CoverageDump, "src/tools/coverage-dump", "coverage-dump"; -); - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] -pub struct ErrorIndex { - pub compiler: Compiler, -} - -impl ErrorIndex { - pub fn command(builder: &Builder<'_>) -> Command { - // Error-index-generator links with the rustdoc library, so we need to add `rustc_lib_paths` - // for rustc_private and libLLVM.so, and `sysroot_lib` for libstd, etc. - let host = builder.config.build; - let compiler = builder.compiler_for(builder.top_stage, host, host); - let mut cmd = Command::new(builder.ensure(ErrorIndex { compiler })); - let mut dylib_paths = builder.rustc_lib_paths(compiler); - dylib_paths.push(PathBuf::from(&builder.sysroot_libdir(compiler, compiler.host))); - add_dylib_path(dylib_paths, &mut cmd); - cmd - } -} - -impl Step for ErrorIndex { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/error_index_generator") - } - - fn make_run(run: RunConfig<'_>) { - // Compile the error-index in the same stage as rustdoc to avoid - // recompiling rustdoc twice if we can. - // - // NOTE: This `make_run` isn't used in normal situations, only if you - // manually build the tool with `x.py build - // src/tools/error-index-generator` which almost nobody does. - // Normally, `x.py test` or `x.py doc` will use the - // `ErrorIndex::command` function instead. - let compiler = - run.builder.compiler(run.builder.top_stage.saturating_sub(1), run.builder.config.build); - run.builder.ensure(ErrorIndex { compiler }); - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - builder - .ensure(ToolBuild { - compiler: self.compiler, - target: self.compiler.host, - tool: "error_index_generator", - mode: Mode::ToolRustc, - path: "src/tools/error_index_generator", - is_optional_tool: false, - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - }) - .expect("expected to build -- essential tool") - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct RemoteTestServer { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for RemoteTestServer { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/remote-test-server") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RemoteTestServer { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - builder - .ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "remote-test-server", - mode: Mode::ToolStd, - path: "src/tools/remote-test-server", - is_optional_tool: false, - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - }) - .expect("expected to build -- essential tool") - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] -pub struct Rustdoc { - /// This should only ever be 0 or 2. - /// We sometimes want to reference the "bootstrap" rustdoc, which is why this option is here. - pub compiler: Compiler, -} - -impl Step for Rustdoc { - type Output = PathBuf; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/rustdoc").path("src/librustdoc") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Rustdoc { - // Note: this is somewhat unique in that we actually want a *target* - // compiler here, because rustdoc *is* a compiler. We won't be using - // this as the compiler to build with, but rather this is "what - // compiler are we producing"? - compiler: run.builder.compiler(run.builder.top_stage, run.target), - }); - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - let target_compiler = self.compiler; - if target_compiler.stage == 0 { - if !target_compiler.is_snapshot(builder) { - panic!("rustdoc in stage 0 must be snapshot rustdoc"); - } - return builder.initial_rustc.with_file_name(exe("rustdoc", target_compiler.host)); - } - let target = target_compiler.host; - // Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise - // we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage - // compilers, which isn't what we want. Rustdoc should be linked in the same way as the - // rustc compiler it's paired with, so it must be built with the previous stage compiler. - let build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build); - - // When using `download-rustc` and a stage0 build_compiler, copying rustc doesn't actually - // build stage0 libstd (because the libstd in sysroot has the wrong ABI). Explicitly build - // it. - builder.ensure(compile::Std::new(build_compiler, target_compiler.host)); - builder.ensure(compile::Rustc::new(build_compiler, target_compiler.host)); - // NOTE: this implies that `download-rustc` is pretty useless when compiling with the stage0 - // compiler, since you do just as much work. - if !builder.config.dry_run() && builder.download_rustc() && build_compiler.stage == 0 { - println!( - "warning: `download-rustc` does nothing when building stage1 tools; consider using `--stage 2` instead" - ); - } - - // The presence of `target_compiler` ensures that the necessary libraries (codegen backends, - // compiler libraries, ...) are built. Rustdoc does not require the presence of any - // libraries within sysroot_libdir (i.e., rustlib), though doctests may want it (since - // they'll be linked to those libraries). As such, don't explicitly `ensure` any additional - // libraries here. The intuition here is that If we've built a compiler, we should be able - // to build rustdoc. - // - let mut features = Vec::new(); - if builder.config.jemalloc { - features.push("jemalloc".to_string()); - } - - let mut cargo = prepare_tool_cargo( - builder, - build_compiler, - Mode::ToolRustc, - target, - "build", - "src/tools/rustdoc", - SourceType::InTree, - features.as_slice(), - ); - - if builder.config.rustc_parallel { - cargo.rustflag("--cfg=parallel_compiler"); - } - - let _guard = builder.msg_tool( - Mode::ToolRustc, - "rustdoc", - build_compiler.stage, - &self.compiler.host, - &target, - ); - builder.run(&mut cargo.into()); - - // Cargo adds a number of paths to the dylib search path on windows, which results in - // the wrong rustdoc being executed. To avoid the conflicting rustdocs, we name the "tool" - // rustdoc a different name. - let tool_rustdoc = builder - .cargo_out(build_compiler, Mode::ToolRustc, target) - .join(exe("rustdoc_tool_binary", target_compiler.host)); - - // don't create a stage0-sysroot/bin directory. - if target_compiler.stage > 0 { - let sysroot = builder.sysroot(target_compiler); - let bindir = sysroot.join("bin"); - t!(fs::create_dir_all(&bindir)); - let bin_rustdoc = bindir.join(exe("rustdoc", target_compiler.host)); - let _ = fs::remove_file(&bin_rustdoc); - builder.copy(&tool_rustdoc, &bin_rustdoc); - bin_rustdoc - } else { - tool_rustdoc - } - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Cargo { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Cargo { - type Output = PathBuf; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/tools/cargo").default_condition( - builder.config.extended - && builder.config.tools.as_ref().map_or( - true, - // If `tools` is set, search list for this tool. - |tools| tools.iter().any(|tool| tool == "cargo"), - ), - ) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Cargo { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - let cargo_bin_path = builder - .ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "cargo", - mode: Mode::ToolRustc, - path: "src/tools/cargo", - is_optional_tool: false, - source_type: SourceType::Submodule, - extra_features: Vec::new(), - allow_features: "", - }) - .expect("expected to build -- essential tool"); - cargo_bin_path - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct LldWrapper { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for LldWrapper { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - let src_exe = builder - .ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "lld-wrapper", - mode: Mode::ToolStd, - path: "src/tools/lld-wrapper", - is_optional_tool: false, - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - }) - .expect("expected to build -- essential tool"); - - src_exe - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct RustAnalyzer { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl RustAnalyzer { - pub const ALLOW_FEATURES: &'static str = - "proc_macro_internals,proc_macro_diagnostic,proc_macro_span,proc_macro_span_shrink"; -} - -impl Step for RustAnalyzer { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/tools/rust-analyzer").default_condition( - builder.config.extended - && builder - .config - .tools - .as_ref() - .map_or(true, |tools| tools.iter().any(|tool| tool == "rust-analyzer")), - ) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustAnalyzer { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "rust-analyzer", - mode: Mode::ToolStd, - path: "src/tools/rust-analyzer", - extra_features: vec!["rust-analyzer/in-rust-tree".to_owned()], - is_optional_tool: false, - source_type: SourceType::InTree, - allow_features: RustAnalyzer::ALLOW_FEATURES, - }) - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct RustAnalyzerProcMacroSrv { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for RustAnalyzerProcMacroSrv { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - // Allow building `rust-analyzer-proc-macro-srv` both as part of the `rust-analyzer` and as a stand-alone tool. - run.path("src/tools/rust-analyzer") - .path("src/tools/rust-analyzer/crates/proc-macro-srv-cli") - .default_condition(builder.config.tools.as_ref().map_or(true, |tools| { - tools - .iter() - .any(|tool| tool == "rust-analyzer" || tool == "rust-analyzer-proc-macro-srv") - })) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustAnalyzerProcMacroSrv { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let path = builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "rust-analyzer-proc-macro-srv", - mode: Mode::ToolStd, - path: "src/tools/rust-analyzer/crates/proc-macro-srv-cli", - extra_features: vec!["sysroot-abi".to_owned()], - is_optional_tool: false, - source_type: SourceType::InTree, - allow_features: RustAnalyzer::ALLOW_FEATURES, - })?; - - // Copy `rust-analyzer-proc-macro-srv` to `/libexec/` - // so that r-a can use it. - let libexec_path = builder.sysroot(self.compiler).join("libexec"); - t!(fs::create_dir_all(&libexec_path)); - builder.copy(&path, &libexec_path.join("rust-analyzer-proc-macro-srv")); - - Some(path) - } -} - -macro_rules! tool_extended { - (($sel:ident, $builder:ident), - $($name:ident, - $path:expr, - $tool_name:expr, - stable = $stable:expr - $(,tool_std = $tool_std:literal)? - $(,allow_features = $allow_features:expr)? - $(,add_bins_to_sysroot = $add_bins_to_sysroot:expr)? - ;)+) => { - $( - #[derive(Debug, Clone, Hash, PartialEq, Eq)] - pub struct $name { - pub compiler: Compiler, - pub target: TargetSelection, - pub extra_features: Vec, - } - - impl Step for $name { - type Output = Option; - const DEFAULT: bool = true; // Overwritten below - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path($path).default_condition( - builder.config.extended - && builder.config.tools.as_ref().map_or( - // By default, on nightly/dev enable all tools, else only - // build stable tools. - $stable || builder.build.unstable_features(), - // If `tools` is set, search list for this tool. - |tools| { - tools.iter().any(|tool| match tool.as_ref() { - "clippy" => $tool_name == "clippy-driver", - x => $tool_name == x, - }) - }), - ) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure($name { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - extra_features: Vec::new(), - }); - } - - #[allow(unused_mut)] - fn run(mut $sel, $builder: &Builder<'_>) -> Option { - let tool = $builder.ensure(ToolBuild { - compiler: $sel.compiler, - target: $sel.target, - tool: $tool_name, - mode: if false $(|| $tool_std)? { Mode::ToolStd } else { Mode::ToolRustc }, - path: $path, - extra_features: $sel.extra_features, - is_optional_tool: true, - source_type: SourceType::InTree, - allow_features: concat!($($allow_features)*), - })?; - - if (false $(|| !$add_bins_to_sysroot.is_empty())?) && $sel.compiler.stage > 0 { - let bindir = $builder.sysroot($sel.compiler).join("bin"); - t!(fs::create_dir_all(&bindir)); - - #[allow(unused_variables)] - let tools_out = $builder - .cargo_out($sel.compiler, Mode::ToolRustc, $sel.target); - - $(for add_bin in $add_bins_to_sysroot { - let bin_source = tools_out.join(exe(add_bin, $sel.target)); - let bin_destination = bindir.join(exe(add_bin, $sel.compiler.host)); - $builder.copy(&bin_source, &bin_destination); - })? - - let tool = bindir.join(exe($tool_name, $sel.compiler.host)); - Some(tool) - } else { - Some(tool) - } - } - } - )+ - } -} - -// Note: tools need to be also added to `Builder::get_step_descriptions` in `builder.rs` -// to make `./x.py build ` work. -// Note: Most submodule updates for tools are handled by bootstrap.py, since they're needed just to -// invoke Cargo to build bootstrap. See the comment there for more details. -tool_extended!((self, builder), - Cargofmt, "src/tools/rustfmt", "cargo-fmt", stable=true; - CargoClippy, "src/tools/clippy", "cargo-clippy", stable=true; - Clippy, "src/tools/clippy", "clippy-driver", stable=true, add_bins_to_sysroot = ["clippy-driver", "cargo-clippy"]; - Miri, "src/tools/miri", "miri", stable=false, add_bins_to_sysroot = ["miri"]; - CargoMiri, "src/tools/miri/cargo-miri", "cargo-miri", stable=true, add_bins_to_sysroot = ["cargo-miri"]; - // FIXME: tool_std is not quite right, we shouldn't allow nightly features. - // But `builder.cargo` doesn't know how to handle ToolBootstrap in stages other than 0, - // and this is close enough for now. - Rls, "src/tools/rls", "rls", stable=true, tool_std=true; - RustDemangler, "src/tools/rust-demangler", "rust-demangler", stable=false, tool_std=true; - Rustfmt, "src/tools/rustfmt", "rustfmt", stable=true, add_bins_to_sysroot = ["rustfmt", "cargo-fmt"]; -); - -impl<'a> Builder<'a> { - /// Gets a `Command` which is ready to run `tool` in `stage` built for - /// `host`. - pub fn tool_cmd(&self, tool: Tool) -> Command { - let mut cmd = Command::new(self.tool_exe(tool)); - let compiler = self.compiler(0, self.config.build); - let host = &compiler.host; - // Prepares the `cmd` provided to be able to run the `compiler` provided. - // - // Notably this munges the dynamic library lookup path to point to the - // right location to run `compiler`. - let mut lib_paths: Vec = vec![ - self.build.rustc_snapshot_libdir(), - self.cargo_out(compiler, Mode::ToolBootstrap, *host).join("deps"), - ]; - - // On MSVC a tool may invoke a C compiler (e.g., compiletest in run-make - // mode) and that C compiler may need some extra PATH modification. Do - // so here. - if compiler.host.contains("msvc") { - let curpaths = env::var_os("PATH").unwrap_or_default(); - let curpaths = env::split_paths(&curpaths).collect::>(); - for &(ref k, ref v) in self.cc.borrow()[&compiler.host].env() { - if k != "PATH" { - continue; - } - for path in env::split_paths(v) { - if !curpaths.contains(&path) { - lib_paths.push(path); - } - } - } - } - - add_dylib_path(lib_paths, &mut cmd); - - // Provide a RUSTC for this command to use. - cmd.env("RUSTC", &self.initial_rustc); - - cmd - } -} diff --git a/src/bootstrap/toolstate.rs b/src/bootstrap/toolstate.rs deleted file mode 100644 index 308023537..000000000 --- a/src/bootstrap/toolstate.rs +++ /dev/null @@ -1,478 +0,0 @@ -use crate::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::util::t; -use serde_derive::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::env; -use std::fmt; -use std::fs; -use std::io::{Seek, SeekFrom}; -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::time; - -// Each cycle is 42 days long (6 weeks); the last week is 35..=42 then. -const BETA_WEEK_START: u64 = 35; - -#[cfg(target_os = "linux")] -const OS: Option<&str> = Some("linux"); - -#[cfg(windows)] -const OS: Option<&str> = Some("windows"); - -#[cfg(all(not(target_os = "linux"), not(windows)))] -const OS: Option<&str> = None; - -type ToolstateData = HashMap, ToolState>; - -#[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, PartialOrd)] -#[serde(rename_all = "kebab-case")] -/// Whether a tool can be compiled, tested or neither -pub enum ToolState { - /// The tool compiles successfully, but the test suite fails - TestFail = 1, - /// The tool compiles successfully and its test suite passes - TestPass = 2, - /// The tool can't even be compiled - BuildFail = 0, -} - -impl fmt::Display for ToolState { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "{}", - match self { - ToolState::TestFail => "test-fail", - ToolState::TestPass => "test-pass", - ToolState::BuildFail => "build-fail", - } - ) - } -} - -/// Number of days after the last promotion of beta. -/// Its value is 41 on the Tuesday where "Promote master to beta (T-2)" happens. -/// The Wednesday after this has value 0. -/// We track this value to prevent regressing tools in the last week of the 6-week cycle. -fn days_since_beta_promotion() -> u64 { - let since_epoch = t!(time::SystemTime::UNIX_EPOCH.elapsed()); - (since_epoch.as_secs() / 86400 - 20) % 42 -} - -// These tools must test-pass on the beta/stable channels. -// -// On the nightly channel, their build step must be attempted, but they may not -// be able to build successfully. -static STABLE_TOOLS: &[(&str, &str)] = &[ - ("book", "src/doc/book"), - ("nomicon", "src/doc/nomicon"), - ("reference", "src/doc/reference"), - ("rust-by-example", "src/doc/rust-by-example"), - ("edition-guide", "src/doc/edition-guide"), -]; - -// These tools are permitted to not build on the beta/stable channels. -// -// We do require that we checked whether they build or not on the tools builder, -// though, as otherwise we will be unable to file an issue if they start -// failing. -static NIGHTLY_TOOLS: &[(&str, &str)] = &[ - ("embedded-book", "src/doc/embedded-book"), - // ("rustc-dev-guide", "src/doc/rustc-dev-guide"), -]; - -fn print_error(tool: &str, submodule: &str) { - eprintln!(); - eprintln!("We detected that this PR updated '{tool}', but its tests failed."); - eprintln!(); - eprintln!("If you do intend to update '{tool}', please check the error messages above and"); - eprintln!("commit another update."); - eprintln!(); - eprintln!("If you do NOT intend to update '{tool}', please ensure you did not accidentally"); - eprintln!("change the submodule at '{submodule}'. You may ask your reviewer for the"); - eprintln!("proper steps."); - crate::exit!(3); -} - -fn check_changed_files(toolstates: &HashMap, ToolState>) { - // Changed files - let output = std::process::Command::new("git") - .arg("diff") - .arg("--name-status") - .arg("HEAD") - .arg("HEAD^") - .output(); - let output = match output { - Ok(o) => o, - Err(e) => { - eprintln!("Failed to get changed files: {e:?}"); - crate::exit!(1); - } - }; - - let output = t!(String::from_utf8(output.stdout)); - - for (tool, submodule) in STABLE_TOOLS.iter().chain(NIGHTLY_TOOLS.iter()) { - let changed = output.lines().any(|l| l.starts_with('M') && l.ends_with(submodule)); - eprintln!("Verifying status of {tool}..."); - if !changed { - continue; - } - - eprintln!("This PR updated '{submodule}', verifying if status is 'test-pass'..."); - if toolstates[*tool] != ToolState::TestPass { - print_error(tool, submodule); - } - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct ToolStateCheck; - -impl Step for ToolStateCheck { - type Output = (); - - /// Checks tool state status. - /// - /// This is intended to be used in the `checktools.sh` script. To use - /// this, set `save-toolstates` in `config.toml` so that tool status will - /// be saved to a JSON file. Then, run `x.py test --no-fail-fast` for all - /// of the tools to populate the JSON file. After that is done, this - /// command can be run to check for any status failures, and exits with an - /// error if there are any. - /// - /// This also handles publishing the results to the `history` directory of - /// the toolstate repo - /// if the env var `TOOLSTATE_PUBLISH` is set. Note that there is a - /// *separate* step of updating the `latest.json` file and creating GitHub - /// issues and comments in `src/ci/publish_toolstate.sh`, which is only - /// performed on master. (The shell/python code is intended to be migrated - /// here eventually.) - /// - /// The rules for failure are: - /// * If the PR modifies a tool, the status must be test-pass. - /// NOTE: There is intent to change this, see - /// . - /// * All "stable" tools must be test-pass on the stable or beta branches. - /// * During beta promotion week, a PR is not allowed to "regress" a - /// stable tool. That is, the status is not allowed to get worse - /// (test-pass to test-fail or build-fail). - fn run(self, builder: &Builder<'_>) { - if builder.config.dry_run() { - return; - } - - let days_since_beta_promotion = days_since_beta_promotion(); - let in_beta_week = days_since_beta_promotion >= BETA_WEEK_START; - let is_nightly = !(builder.config.channel == "beta" || builder.config.channel == "stable"); - let toolstates = builder.toolstates(); - - let mut did_error = false; - - for (tool, _) in STABLE_TOOLS.iter().chain(NIGHTLY_TOOLS.iter()) { - if !toolstates.contains_key(*tool) { - did_error = true; - eprintln!("error: Tool `{tool}` was not recorded in tool state."); - } - } - - if did_error { - crate::exit!(1); - } - - check_changed_files(&toolstates); - checkout_toolstate_repo(); - let old_toolstate = read_old_toolstate(); - - for (tool, _) in STABLE_TOOLS.iter() { - let state = toolstates[*tool]; - - if state != ToolState::TestPass { - if !is_nightly { - did_error = true; - eprintln!("error: Tool `{tool}` should be test-pass but is {state}"); - } else if in_beta_week { - let old_state = old_toolstate - .iter() - .find(|ts| ts.tool == *tool) - .expect("latest.json missing tool") - .state(); - if state < old_state { - did_error = true; - eprintln!( - "error: Tool `{tool}` has regressed from {old_state} to {state} during beta week." - ); - } else { - // This warning only appears in the logs, which most - // people won't read. It's mostly here for testing and - // debugging. - eprintln!( - "warning: Tool `{tool}` is not test-pass (is `{state}`), \ - this should be fixed before beta is branched." - ); - } - } - // `publish_toolstate.py` is responsible for updating - // `latest.json` and creating comments/issues warning people - // if there is a regression. That all happens in a separate CI - // job on the master branch once the PR has passed all tests - // on the `auto` branch. - } - } - - if did_error { - crate::exit!(1); - } - - if builder.config.channel == "nightly" && env::var_os("TOOLSTATE_PUBLISH").is_some() { - commit_toolstate_change(&toolstates); - } - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("check-tools") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(ToolStateCheck); - } -} - -impl Builder<'_> { - fn toolstates(&self) -> HashMap, ToolState> { - if let Some(ref path) = self.config.save_toolstates { - if let Some(parent) = path.parent() { - // Ensure the parent directory always exists - t!(std::fs::create_dir_all(parent)); - } - let mut file = - t!(fs::OpenOptions::new().create(true).write(true).read(true).open(path)); - - serde_json::from_reader(&mut file).unwrap_or_default() - } else { - Default::default() - } - } - - /// Updates the actual toolstate of a tool. - /// - /// The toolstates are saved to the file specified by the key - /// `rust.save-toolstates` in `config.toml`. If unspecified, nothing will be - /// done. The file is updated immediately after this function completes. - pub fn save_toolstate(&self, tool: &str, state: ToolState) { - use std::io::Write; - - // If we're in a dry run setting we don't want to save toolstates as - // that means if we e.g. panic down the line it'll look like we tested - // everything (but we actually haven't). - if self.config.dry_run() { - return; - } - // Toolstate isn't tracked for clippy or rustfmt, but since most tools do, we avoid checking - // in all the places we could save toolstate and just do so here. - if tool == "clippy-driver" || tool == "rustfmt" { - return; - } - if let Some(ref path) = self.config.save_toolstates { - if let Some(parent) = path.parent() { - // Ensure the parent directory always exists - t!(std::fs::create_dir_all(parent)); - } - let mut file = - t!(fs::OpenOptions::new().create(true).read(true).write(true).open(path)); - - let mut current_toolstates: HashMap, ToolState> = - serde_json::from_reader(&mut file).unwrap_or_default(); - current_toolstates.insert(tool.into(), state); - t!(file.seek(SeekFrom::Start(0))); - t!(file.set_len(0)); - t!(serde_json::to_writer(&file, ¤t_toolstates)); - t!(writeln!(file)); // make sure this ends in a newline - } - } -} - -fn toolstate_repo() -> String { - env::var("TOOLSTATE_REPO") - .unwrap_or_else(|_| "https://github.com/rust-lang-nursery/rust-toolstate.git".to_string()) -} - -/// Directory where the toolstate repo is checked out. -const TOOLSTATE_DIR: &str = "rust-toolstate"; - -/// Checks out the toolstate repo into `TOOLSTATE_DIR`. -fn checkout_toolstate_repo() { - if let Ok(token) = env::var("TOOLSTATE_REPO_ACCESS_TOKEN") { - prepare_toolstate_config(&token); - } - if Path::new(TOOLSTATE_DIR).exists() { - eprintln!("Cleaning old toolstate directory..."); - t!(fs::remove_dir_all(TOOLSTATE_DIR)); - } - - let status = Command::new("git") - .arg("clone") - .arg("--depth=1") - .arg(toolstate_repo()) - .arg(TOOLSTATE_DIR) - .status(); - let success = match status { - Ok(s) => s.success(), - Err(_) => false, - }; - if !success { - panic!("git clone unsuccessful (status: {status:?})"); - } -} - -/// Sets up config and authentication for modifying the toolstate repo. -fn prepare_toolstate_config(token: &str) { - fn git_config(key: &str, value: &str) { - let status = Command::new("git").arg("config").arg("--global").arg(key).arg(value).status(); - let success = match status { - Ok(s) => s.success(), - Err(_) => false, - }; - if !success { - panic!("git config key={key} value={value} failed (status: {status:?})"); - } - } - - // If changing anything here, then please check that `src/ci/publish_toolstate.sh` is up to date - // as well. - git_config("user.email", "7378925+rust-toolstate-update@users.noreply.github.com"); - git_config("user.name", "Rust Toolstate Update"); - git_config("credential.helper", "store"); - - let credential = format!("https://{token}:x-oauth-basic@github.com\n",); - let git_credential_path = PathBuf::from(t!(env::var("HOME"))).join(".git-credentials"); - t!(fs::write(&git_credential_path, credential)); -} - -/// Reads the latest toolstate from the toolstate repo. -fn read_old_toolstate() -> Vec { - let latest_path = Path::new(TOOLSTATE_DIR).join("_data").join("latest.json"); - let old_toolstate = t!(fs::read(latest_path)); - t!(serde_json::from_slice(&old_toolstate)) -} - -/// This function `commit_toolstate_change` provides functionality for pushing a change -/// to the `rust-toolstate` repository. -/// -/// The function relies on a GitHub bot user, which should have a Personal access -/// token defined in the environment variable $TOOLSTATE_REPO_ACCESS_TOKEN. If for -/// some reason you need to change the token, please update the Azure Pipelines -/// variable group. -/// -/// 1. Generate a new Personal access token: -/// -/// * Login to the bot account, and go to Settings -> Developer settings -> -/// Personal access tokens -/// * Click "Generate new token" -/// * Enable the "public_repo" permission, then click "Generate token" -/// * Copy the generated token (should be a 40-digit hexadecimal number). -/// Save it somewhere secure, as the token would be gone once you leave -/// the page. -/// -/// 2. Update the variable group in Azure Pipelines -/// -/// * Ping a member of the infrastructure team to do this. -/// -/// 4. Replace the email address below if the bot account identity is changed -/// -/// * See -/// if a private email by GitHub is wanted. -fn commit_toolstate_change(current_toolstate: &ToolstateData) { - let message = format!("({} CI update)", OS.expect("linux/windows only")); - let mut success = false; - for _ in 1..=5 { - // Upload the test results (the new commit-to-toolstate mapping) to the toolstate repo. - // This does *not* change the "current toolstate"; that only happens post-landing - // via `src/ci/docker/publish_toolstate.sh`. - publish_test_results(¤t_toolstate); - - // `git commit` failing means nothing to commit. - let status = t!(Command::new("git") - .current_dir(TOOLSTATE_DIR) - .arg("commit") - .arg("-a") - .arg("-m") - .arg(&message) - .status()); - if !status.success() { - success = true; - break; - } - - let status = t!(Command::new("git") - .current_dir(TOOLSTATE_DIR) - .arg("push") - .arg("origin") - .arg("master") - .status()); - // If we successfully push, exit. - if status.success() { - success = true; - break; - } - eprintln!("Sleeping for 3 seconds before retrying push"); - std::thread::sleep(std::time::Duration::from_secs(3)); - let status = t!(Command::new("git") - .current_dir(TOOLSTATE_DIR) - .arg("fetch") - .arg("origin") - .arg("master") - .status()); - assert!(status.success()); - let status = t!(Command::new("git") - .current_dir(TOOLSTATE_DIR) - .arg("reset") - .arg("--hard") - .arg("origin/master") - .status()); - assert!(status.success()); - } - - if !success { - panic!("Failed to update toolstate repository with new data"); - } -} - -/// Updates the "history" files with the latest results. -/// -/// These results will later be promoted to `latest.json` by the -/// `publish_toolstate.py` script if the PR passes all tests and is merged to -/// master. -fn publish_test_results(current_toolstate: &ToolstateData) { - let commit = t!(std::process::Command::new("git").arg("rev-parse").arg("HEAD").output()); - let commit = t!(String::from_utf8(commit.stdout)); - - let toolstate_serialized = t!(serde_json::to_string(¤t_toolstate)); - - let history_path = Path::new(TOOLSTATE_DIR) - .join("history") - .join(format!("{}.tsv", OS.expect("linux/windows only"))); - let mut file = t!(fs::read_to_string(&history_path)); - let end_of_first_line = file.find('\n').unwrap(); - file.insert_str(end_of_first_line, &format!("\n{}\t{}", commit.trim(), toolstate_serialized)); - t!(fs::write(&history_path, file)); -} - -#[derive(Debug, Deserialize)] -struct RepoState { - tool: String, - windows: ToolState, - linux: ToolState, -} - -impl RepoState { - fn state(&self) -> ToolState { - if cfg!(target_os = "linux") { - self.linux - } else if cfg!(windows) { - self.windows - } else { - unimplemented!() - } - } -} diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs deleted file mode 100644 index 3c4a21434..000000000 --- a/src/bootstrap/util.rs +++ /dev/null @@ -1,497 +0,0 @@ -//! Various utility functions used throughout rustbuild. -//! -//! Simple things like testing the various filesystem operations here and there, -//! not a lot of interesting happenings here unfortunately. - -use build_helper::util::{fail, try_run}; -use std::env; -use std::fs; -use std::io; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use std::str; -use std::time::{Instant, SystemTime, UNIX_EPOCH}; - -use crate::builder::Builder; -use crate::config::{Config, TargetSelection}; -use crate::OnceCell; - -/// A helper macro to `unwrap` a result except also print out details like: -/// -/// * The file/line of the panic -/// * The expression that failed -/// * The error itself -/// -/// This is currently used judiciously throughout the build system rather than -/// using a `Result` with `try!`, but this may change one day... -#[macro_export] -macro_rules! t { - ($e:expr) => { - match $e { - Ok(e) => e, - Err(e) => panic!("{} failed with {}", stringify!($e), e), - } - }; - // it can show extra info in the second parameter - ($e:expr, $extra:expr) => { - match $e { - Ok(e) => e, - Err(e) => panic!("{} failed with {} ({:?})", stringify!($e), e, $extra), - } - }; -} -pub use t; - -/// Given an executable called `name`, return the filename for the -/// executable for a particular target. -pub fn exe(name: &str, target: TargetSelection) -> String { - if target.contains("windows") { - format!("{name}.exe") - } else if target.contains("uefi") { - format!("{name}.efi") - } else { - name.to_string() - } -} - -/// Returns `true` if the file name given looks like a dynamic library. -pub fn is_dylib(name: &str) -> bool { - name.ends_with(".dylib") || name.ends_with(".so") || name.ends_with(".dll") -} - -/// Returns `true` if the file name given looks like a debug info file -pub fn is_debug_info(name: &str) -> bool { - // FIXME: consider split debug info on other platforms (e.g., Linux, macOS) - name.ends_with(".pdb") -} - -/// Returns the corresponding relative library directory that the compiler's -/// dylibs will be found in. -pub fn libdir(target: TargetSelection) -> &'static str { - if target.contains("windows") { "bin" } else { "lib" } -} - -/// Adds a list of lookup paths to `cmd`'s dynamic library lookup path. -/// If the dylib_path_var is already set for this cmd, the old value will be overwritten! -pub fn add_dylib_path(path: Vec, cmd: &mut Command) { - let mut list = dylib_path(); - for path in path { - list.insert(0, path); - } - cmd.env(dylib_path_var(), t!(env::join_paths(list))); -} - -include!("dylib_util.rs"); - -/// Adds a list of lookup paths to `cmd`'s link library lookup path. -pub fn add_link_lib_path(path: Vec, cmd: &mut Command) { - let mut list = link_lib_path(); - for path in path { - list.insert(0, path); - } - cmd.env(link_lib_path_var(), t!(env::join_paths(list))); -} - -/// Returns the environment variable which the link library lookup path -/// resides in for this platform. -fn link_lib_path_var() -> &'static str { - if cfg!(target_env = "msvc") { "LIB" } else { "LIBRARY_PATH" } -} - -/// Parses the `link_lib_path_var()` environment variable, returning a list of -/// paths that are members of this lookup path. -fn link_lib_path() -> Vec { - let var = match env::var_os(link_lib_path_var()) { - Some(v) => v, - None => return vec![], - }; - env::split_paths(&var).collect() -} - -pub struct TimeIt(bool, Instant); - -/// Returns an RAII structure that prints out how long it took to drop. -pub fn timeit(builder: &Builder<'_>) -> TimeIt { - TimeIt(builder.config.dry_run(), Instant::now()) -} - -impl Drop for TimeIt { - fn drop(&mut self) { - let time = self.1.elapsed(); - if !self.0 { - println!("\tfinished in {}.{:03} seconds", time.as_secs(), time.subsec_millis()); - } - } -} - -/// Used for download caching -pub(crate) fn program_out_of_date(stamp: &Path, key: &str) -> bool { - if !stamp.exists() { - return true; - } - t!(fs::read_to_string(stamp)) != key -} - -/// Symlinks two directories, using junctions on Windows and normal symlinks on -/// Unix. -pub fn symlink_dir(config: &Config, original: &Path, link: &Path) -> io::Result<()> { - if config.dry_run() { - return Ok(()); - } - let _ = fs::remove_dir(link); - return symlink_dir_inner(original, link); - - #[cfg(not(windows))] - fn symlink_dir_inner(original: &Path, link: &Path) -> io::Result<()> { - use std::os::unix::fs; - fs::symlink(original, link) - } - - #[cfg(windows)] - fn symlink_dir_inner(target: &Path, junction: &Path) -> io::Result<()> { - junction::create(&target, &junction) - } -} - -pub fn forcing_clang_based_tests() -> bool { - if let Some(var) = env::var_os("RUSTBUILD_FORCE_CLANG_BASED_TESTS") { - match &var.to_string_lossy().to_lowercase()[..] { - "1" | "yes" | "on" => true, - "0" | "no" | "off" => false, - other => { - // Let's make sure typos don't go unnoticed - panic!( - "Unrecognized option '{other}' set in \ - RUSTBUILD_FORCE_CLANG_BASED_TESTS" - ) - } - } - } else { - false - } -} - -pub fn use_host_linker(target: TargetSelection) -> bool { - // FIXME: this information should be gotten by checking the linker flavor - // of the rustc target - !(target.contains("emscripten") - || target.contains("wasm32") - || target.contains("nvptx") - || target.contains("fortanix") - || target.contains("fuchsia") - || target.contains("bpf") - || target.contains("switch")) -} - -pub fn is_valid_test_suite_arg<'a, P: AsRef>( - path: &'a Path, - suite_path: P, - builder: &Builder<'_>, -) -> Option<&'a str> { - let suite_path = suite_path.as_ref(); - let path = match path.strip_prefix(".") { - Ok(p) => p, - Err(_) => path, - }; - if !path.starts_with(suite_path) { - return None; - } - let abs_path = builder.src.join(path); - let exists = abs_path.is_dir() || abs_path.is_file(); - if !exists { - panic!( - "Invalid test suite filter \"{}\": file or directory does not exist", - abs_path.display() - ); - } - // Since test suite paths are themselves directories, if we don't - // specify a directory or file, we'll get an empty string here - // (the result of the test suite directory without its suite prefix). - // Therefore, we need to filter these out, as only the first --test-args - // flag is respected, so providing an empty --test-args conflicts with - // any following it. - match path.strip_prefix(suite_path).ok().and_then(|p| p.to_str()) { - Some(s) if !s.is_empty() => Some(s), - _ => None, - } -} - -pub fn run(cmd: &mut Command, print_cmd_on_fail: bool) { - if try_run(cmd, print_cmd_on_fail).is_err() { - crate::exit!(1); - } -} - -pub fn check_run(cmd: &mut Command, print_cmd_on_fail: bool) -> bool { - let status = match cmd.status() { - Ok(status) => status, - Err(e) => { - println!("failed to execute command: {cmd:?}\nerror: {e}"); - return false; - } - }; - if !status.success() && print_cmd_on_fail { - println!( - "\n\ncommand did not execute successfully: {cmd:?}\n\ - expected success, got: {status}\n\n" - ); - } - status.success() -} - -pub fn run_suppressed(cmd: &mut Command) { - if !try_run_suppressed(cmd) { - crate::exit!(1); - } -} - -pub fn try_run_suppressed(cmd: &mut Command) -> bool { - let output = match cmd.output() { - Ok(status) => status, - Err(e) => fail(&format!("failed to execute command: {cmd:?}\nerror: {e}")), - }; - if !output.status.success() { - println!( - "\n\ncommand did not execute successfully: {:?}\n\ - expected success, got: {}\n\n\ - stdout ----\n{}\n\ - stderr ----\n{}\n\n", - cmd, - output.status, - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - } - output.status.success() -} - -pub fn make(host: &str) -> PathBuf { - if host.contains("dragonfly") - || host.contains("freebsd") - || host.contains("netbsd") - || host.contains("openbsd") - { - PathBuf::from("gmake") - } else { - PathBuf::from("make") - } -} - -#[track_caller] -pub fn output(cmd: &mut Command) -> String { - let output = match cmd.stderr(Stdio::inherit()).output() { - Ok(status) => status, - Err(e) => fail(&format!("failed to execute command: {cmd:?}\nerror: {e}")), - }; - if !output.status.success() { - panic!( - "command did not execute successfully: {:?}\n\ - expected success, got: {}", - cmd, output.status - ); - } - String::from_utf8(output.stdout).unwrap() -} - -pub fn output_result(cmd: &mut Command) -> Result { - let output = match cmd.stderr(Stdio::inherit()).output() { - Ok(status) => status, - Err(e) => return Err(format!("failed to run command: {cmd:?}: {e}")), - }; - if !output.status.success() { - return Err(format!( - "command did not execute successfully: {:?}\n\ - expected success, got: {}\n{}", - cmd, - output.status, - String::from_utf8(output.stderr).map_err(|err| format!("{err:?}"))? - )); - } - Ok(String::from_utf8(output.stdout).map_err(|err| format!("{err:?}"))?) -} - -/// Returns the last-modified time for `path`, or zero if it doesn't exist. -pub fn mtime(path: &Path) -> SystemTime { - fs::metadata(path).and_then(|f| f.modified()).unwrap_or(UNIX_EPOCH) -} - -/// Returns `true` if `dst` is up to date given that the file or files in `src` -/// are used to generate it. -/// -/// Uses last-modified time checks to verify this. -pub fn up_to_date(src: &Path, dst: &Path) -> bool { - if !dst.exists() { - return false; - } - let threshold = mtime(dst); - let meta = match fs::metadata(src) { - Ok(meta) => meta, - Err(e) => panic!("source {src:?} failed to get metadata: {e}"), - }; - if meta.is_dir() { - dir_up_to_date(src, threshold) - } else { - meta.modified().unwrap_or(UNIX_EPOCH) <= threshold - } -} - -fn dir_up_to_date(src: &Path, threshold: SystemTime) -> bool { - t!(fs::read_dir(src)).map(|e| t!(e)).all(|e| { - let meta = t!(e.metadata()); - if meta.is_dir() { - dir_up_to_date(&e.path(), threshold) - } else { - meta.modified().unwrap_or(UNIX_EPOCH) < threshold - } - }) -} - -/// Copied from `std::path::absolute` until it stabilizes. -/// -/// FIXME: this shouldn't exist. -pub(crate) fn absolute(path: &Path) -> PathBuf { - if path.as_os_str().is_empty() { - panic!("can't make empty path absolute"); - } - #[cfg(unix)] - { - t!(absolute_unix(path), format!("could not make path absolute: {}", path.display())) - } - #[cfg(windows)] - { - t!(absolute_windows(path), format!("could not make path absolute: {}", path.display())) - } - #[cfg(not(any(unix, windows)))] - { - println!("warning: bootstrap is not supported on non-unix platforms"); - t!(std::fs::canonicalize(t!(std::env::current_dir()))).join(path) - } -} - -#[cfg(unix)] -/// Make a POSIX path absolute without changing its semantics. -fn absolute_unix(path: &Path) -> io::Result { - // This is mostly a wrapper around collecting `Path::components`, with - // exceptions made where this conflicts with the POSIX specification. - // See 4.13 Pathname Resolution, IEEE Std 1003.1-2017 - // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 - - use std::os::unix::prelude::OsStrExt; - let mut components = path.components(); - let path_os = path.as_os_str().as_bytes(); - - let mut normalized = if path.is_absolute() { - // "If a pathname begins with two successive characters, the - // first component following the leading characters may be - // interpreted in an implementation-defined manner, although more than - // two leading characters shall be treated as a single - // character." - if path_os.starts_with(b"//") && !path_os.starts_with(b"///") { - components.next(); - PathBuf::from("//") - } else { - PathBuf::new() - } - } else { - env::current_dir()? - }; - normalized.extend(components); - - // "Interfaces using pathname resolution may specify additional constraints - // when a pathname that does not name an existing directory contains at - // least one non- character and contains one or more trailing - // characters". - // A trailing is also meaningful if "a symbolic link is - // encountered during pathname resolution". - - if path_os.ends_with(b"/") { - normalized.push(""); - } - - Ok(normalized) -} - -#[cfg(windows)] -fn absolute_windows(path: &std::path::Path) -> std::io::Result { - use std::ffi::OsString; - use std::io::Error; - use std::os::windows::ffi::{OsStrExt, OsStringExt}; - use std::ptr::null_mut; - #[link(name = "kernel32")] - extern "system" { - fn GetFullPathNameW( - lpFileName: *const u16, - nBufferLength: u32, - lpBuffer: *mut u16, - lpFilePart: *mut *const u16, - ) -> u32; - } - - unsafe { - // encode the path as UTF-16 - let path: Vec = path.as_os_str().encode_wide().chain([0]).collect(); - let mut buffer = Vec::new(); - // Loop until either success or failure. - loop { - // Try to get the absolute path - let len = GetFullPathNameW( - path.as_ptr(), - buffer.len().try_into().unwrap(), - buffer.as_mut_ptr(), - null_mut(), - ); - match len as usize { - // Failure - 0 => return Err(Error::last_os_error()), - // Buffer is too small, resize. - len if len > buffer.len() => buffer.resize(len, 0), - // Success! - len => { - buffer.truncate(len); - return Ok(OsString::from_wide(&buffer).into()); - } - } - } - } -} - -/// Adapted from -/// -/// When `clang-cl` is used with instrumentation, we need to add clang's runtime library resource -/// directory to the linker flags, otherwise there will be linker errors about the profiler runtime -/// missing. This function returns the path to that directory. -pub fn get_clang_cl_resource_dir(clang_cl_path: &str) -> PathBuf { - // Similar to how LLVM does it, to find clang's library runtime directory: - // - we ask `clang-cl` to locate the `clang_rt.builtins` lib. - let mut builtins_locator = Command::new(clang_cl_path); - builtins_locator.args(&["/clang:-print-libgcc-file-name", "/clang:--rtlib=compiler-rt"]); - - let clang_rt_builtins = output(&mut builtins_locator); - let clang_rt_builtins = Path::new(clang_rt_builtins.trim()); - assert!( - clang_rt_builtins.exists(), - "`clang-cl` must correctly locate the library runtime directory" - ); - - // - the profiler runtime will be located in the same directory as the builtins lib, like - // `$LLVM_DISTRO_ROOT/lib/clang/$LLVM_VERSION/lib/windows`. - let clang_rt_dir = clang_rt_builtins.parent().expect("The clang lib folder should exist"); - clang_rt_dir.to_path_buf() -} - -pub fn lld_flag_no_threads(is_windows: bool) -> &'static str { - static LLD_NO_THREADS: OnceCell<(&'static str, &'static str)> = OnceCell::new(); - let (windows, other) = LLD_NO_THREADS.get_or_init(|| { - let out = output(Command::new("lld").arg("-flavor").arg("ld").arg("--version")); - let newer = match (out.find(char::is_numeric), out.find('.')) { - (Some(b), Some(e)) => out.as_str()[b..e].parse::().ok().unwrap_or(14) > 10, - _ => true, - }; - if newer { ("/threads:1", "--threads=1") } else { ("/no-threads", "--no-threads") } - }); - if is_windows { windows } else { other } -} - -pub fn dir_is_empty(dir: &Path) -> bool { - t!(std::fs::read_dir(dir)).next().is_none() -} -- cgit v1.2.3