diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-06-07 05:48:42 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-06-07 05:48:42 +0000 |
commit | cec1877e180393eba0f6ddb0cf97bf3a791631c7 (patch) | |
tree | 47b4dac2a9dd9a40c30c251b4d4a72d7ccf77e9f /src/bootstrap | |
parent | Adding debian version 1.74.1+dfsg1-1. (diff) | |
download | rustc-cec1877e180393eba0f6ddb0cf97bf3a791631c7.tar.xz rustc-cec1877e180393eba0f6ddb0cf97bf3a791631c7.zip |
Merging upstream version 1.75.0+dfsg1.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/bootstrap')
-rw-r--r-- | src/bootstrap/CHANGELOG.md | 71 | ||||
-rw-r--r-- | src/bootstrap/Cargo.lock | 181 | ||||
-rw-r--r-- | src/bootstrap/Cargo.toml | 39 | ||||
-rw-r--r-- | src/bootstrap/README.md | 14 | ||||
-rw-r--r-- | src/bootstrap/bootstrap.py | 29 | ||||
-rw-r--r-- | src/bootstrap/bootstrap_test.py | 3 | ||||
-rwxr-xr-x | src/bootstrap/configure.py | 20 | ||||
-rw-r--r-- | src/bootstrap/defaults/config.codegen.toml | 2 | ||||
-rw-r--r-- | src/bootstrap/download-ci-llvm-stamp | 2 | ||||
-rw-r--r-- | src/bootstrap/job.rs | 143 | ||||
-rw-r--r-- | src/bootstrap/src/bin/main.rs (renamed from src/bootstrap/bin/main.rs) | 59 | ||||
-rw-r--r-- | src/bootstrap/src/bin/rustc.rs (renamed from src/bootstrap/bin/rustc.rs) | 72 | ||||
-rw-r--r-- | src/bootstrap/src/bin/rustdoc.rs (renamed from src/bootstrap/bin/rustdoc.rs) | 16 | ||||
-rw-r--r-- | src/bootstrap/src/bin/sccache-plus-cl.rs (renamed from src/bootstrap/bin/sccache-plus-cl.rs) | 0 | ||||
-rw-r--r-- | src/bootstrap/src/core/build_steps/check.rs (renamed from src/bootstrap/check.rs) | 22 | ||||
-rw-r--r-- | src/bootstrap/src/core/build_steps/clean.rs (renamed from src/bootstrap/clean.rs) | 15 | ||||
-rw-r--r-- | src/bootstrap/src/core/build_steps/compile.rs (renamed from src/bootstrap/compile.rs) | 124 | ||||
-rw-r--r-- | src/bootstrap/src/core/build_steps/dist.rs (renamed from src/bootstrap/dist.rs) | 202 | ||||
-rw-r--r-- | src/bootstrap/src/core/build_steps/doc.rs (renamed from src/bootstrap/doc.rs) | 127 | ||||
-rw-r--r-- | src/bootstrap/src/core/build_steps/format.rs (renamed from src/bootstrap/format.rs) | 6 | ||||
-rw-r--r-- | src/bootstrap/src/core/build_steps/install.rs (renamed from src/bootstrap/install.rs) | 28 | ||||
-rw-r--r-- | src/bootstrap/src/core/build_steps/llvm.rs (renamed from src/bootstrap/llvm.rs) | 43 | ||||
-rw-r--r-- | src/bootstrap/src/core/build_steps/mod.rs | 15 | ||||
-rw-r--r-- | src/bootstrap/src/core/build_steps/run.rs (renamed from src/bootstrap/run.rs) | 46 | ||||
-rw-r--r-- | src/bootstrap/src/core/build_steps/setup.rs (renamed from src/bootstrap/setup.rs) | 80 | ||||
-rw-r--r-- | src/bootstrap/src/core/build_steps/suggest.rs (renamed from src/bootstrap/suggest.rs) | 22 | ||||
-rw-r--r-- | src/bootstrap/src/core/build_steps/synthetic_targets.rs (renamed from src/bootstrap/synthetic_targets.rs) | 6 | ||||
-rw-r--r-- | src/bootstrap/src/core/build_steps/test.rs (renamed from src/bootstrap/test.rs) | 400 | ||||
-rw-r--r-- | src/bootstrap/src/core/build_steps/tool.rs (renamed from src/bootstrap/tool.rs) | 25 | ||||
-rw-r--r-- | src/bootstrap/src/core/build_steps/toolstate.rs (renamed from src/bootstrap/toolstate.rs) | 12 | ||||
-rw-r--r-- | src/bootstrap/src/core/builder.rs (renamed from src/bootstrap/builder.rs) | 162 | ||||
-rw-r--r-- | src/bootstrap/src/core/config/config.rs (renamed from src/bootstrap/config.rs) | 198 | ||||
-rw-r--r-- | src/bootstrap/src/core/config/flags.rs (renamed from src/bootstrap/flags.rs) | 14 | ||||
-rw-r--r-- | src/bootstrap/src/core/config/mod.rs | 4 | ||||
-rw-r--r-- | src/bootstrap/src/core/download.rs (renamed from src/bootstrap/download.rs) | 63 | ||||
-rw-r--r-- | src/bootstrap/src/core/metadata.rs (renamed from src/bootstrap/metadata.rs) | 4 | ||||
-rw-r--r-- | src/bootstrap/src/core/mod.rs | 6 | ||||
-rw-r--r-- | src/bootstrap/src/core/sanity.rs (renamed from src/bootstrap/sanity.rs) | 6 | ||||
-rw-r--r-- | src/bootstrap/src/lib.rs (renamed from src/bootstrap/lib.rs) | 338 | ||||
-rw-r--r-- | src/bootstrap/src/tests/builder.rs (renamed from src/bootstrap/builder/tests.rs) | 11 | ||||
-rw-r--r-- | src/bootstrap/src/tests/config.rs (renamed from src/bootstrap/config/tests.rs) | 45 | ||||
-rw-r--r-- | src/bootstrap/src/tests/setup.rs (renamed from src/bootstrap/setup/tests.rs) | 0 | ||||
-rw-r--r-- | src/bootstrap/src/utils/bin_helpers.rs (renamed from src/bootstrap/bin/_helper.rs) | 14 | ||||
-rw-r--r-- | src/bootstrap/src/utils/cache.rs (renamed from src/bootstrap/cache.rs) | 2 | ||||
-rw-r--r-- | src/bootstrap/src/utils/cc_detect.rs (renamed from src/bootstrap/cc_detect.rs) | 83 | ||||
-rw-r--r-- | src/bootstrap/src/utils/channel.rs (renamed from src/bootstrap/channel.rs) | 3 | ||||
-rw-r--r-- | src/bootstrap/src/utils/dylib.rs (renamed from src/bootstrap/dylib_util.rs) | 11 | ||||
-rw-r--r-- | src/bootstrap/src/utils/exec.rs | 60 | ||||
-rw-r--r-- | src/bootstrap/src/utils/helpers.rs (renamed from src/bootstrap/util.rs) | 89 | ||||
-rw-r--r-- | src/bootstrap/src/utils/job.rs | 159 | ||||
-rw-r--r-- | src/bootstrap/src/utils/metrics.rs (renamed from src/bootstrap/metrics.rs) | 6 | ||||
-rw-r--r-- | src/bootstrap/src/utils/mod.rs | 15 | ||||
-rw-r--r-- | src/bootstrap/src/utils/render_tests.rs (renamed from src/bootstrap/render_tests.rs) | 4 | ||||
-rw-r--r-- | src/bootstrap/src/utils/tarball.rs (renamed from src/bootstrap/tarball.rs) | 26 |
54 files changed, 1932 insertions, 1215 deletions
diff --git a/src/bootstrap/CHANGELOG.md b/src/bootstrap/CHANGELOG.md deleted file mode 100644 index 1aba07138..000000000 --- a/src/bootstrap/CHANGELOG.md +++ /dev/null @@ -1,71 +0,0 @@ -# Changelog - -All notable changes to bootstrap will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - - -## [Changes since the last major version] - -- Vendoring is no longer done automatically when building from git sources. To use vendoring, run `cargo vendor` manually, or use the pre-vendored `rustc-src` tarball. -- `llvm-libunwind` now accepts `in-tree` (formerly true), `system` or `no` (formerly false) [#77703](https://github.com/rust-lang/rust/pull/77703) -- The options `infodir`, `localstatedir`, and `gpg-password-file` are no longer allowed in config.toml. Previously, they were ignored without warning. Note that `infodir` and `localstatedir` are still accepted by `./configure`, with a warning. [#82451](https://github.com/rust-lang/rust/pull/82451) -- Change the names for `dist` commands to match the component they generate. [#90684](https://github.com/rust-lang/rust/pull/90684) -- The `build.fast-submodules` option has been removed. Fast submodule checkouts are enabled unconditionally. Automatic submodule handling can still be disabled with `build.submodules = false`. -- Several unsupported `./configure` options have been removed: `optimize`, `parallel-compiler`. These can still be enabled with `--set`, although it isn't recommended. -- `remote-test-server`'s `verbose` argument has been removed in favor of the `--verbose` flag -- `remote-test-server`'s `remote` argument has been removed in favor of the `--bind` flag. Use `--bind 0.0.0.0:12345` to replicate the behavior of the `remote` argument. -- `x.py fmt` now formats only files modified between the merge-base of HEAD and the last commit in the master branch of the rust-lang repository and the current working directory. To restore old behaviour, use `x.py fmt .`. The check mode is not affected by this change. [#105702](https://github.com/rust-lang/rust/pull/105702) -- The `llvm.version-check` config option has been removed. Older versions were never supported. If you still need to support older versions (e.g. you are applying custom patches), patch `check_llvm_version` in bootstrap to change the minimum version. [#108619](https://github.com/rust-lang/rust/pull/108619) -- The `rust.ignore-git` option has been renamed to `rust.omit-git-hash`. [#110059](https://github.com/rust-lang/rust/pull/110059) -- `--exclude` no longer accepts a `Kind` as part of a Step; instead it uses the top-level Kind of the subcommand. If this matches how you were already using --exclude (e.g. `x test --exclude test::std`), simply remove the kind: `--exclude std`. If you were using a kind that did not match the top-level subcommand, please open an issue explaining why you wanted this feature. - -### Non-breaking changes - -- `x.py check` needs opt-in to check tests (--all-targets) [#77473](https://github.com/rust-lang/rust/pull/77473) -- The default bootstrap profiles are now located at `bootstrap/defaults/config.$PROFILE.toml` (previously they were located at `bootstrap/defaults/config.toml.$PROFILE`) [#77558](https://github.com/rust-lang/rust/pull/77558) -- If you have Rust already installed, `x.py` will now infer the host target - from the default rust toolchain. [#78513](https://github.com/rust-lang/rust/pull/78513) -- Add options for enabling overflow checks, one for std (`overflow-checks-std`) and one for everything else (`overflow-checks`). Both default to false. -- Add llvm option `enable-warnings` to have control on llvm compilation warnings. Default to false. -- Add `rpath` option in `target` section to support set rpath option for each target independently. [#111242](https://github.com/rust-lang/rust/pull/111242) - - -## [Version 2] - 2020-09-25 - -- `host` now defaults to the value of `build` in all cases - + Previously `host` defaulted to an empty list when `target` was overridden, and to `build` otherwise - -### Non-breaking changes - -- Add `x.py setup` [#76631](https://github.com/rust-lang/rust/pull/76631) -- Add a changelog for x.py [#76626](https://github.com/rust-lang/rust/pull/76626) -- Optionally, download LLVM from CI on Linux and NixOS. This can be enabled with `download-ci-llvm = true` under `[llvm]`. - + [#76439](https://github.com/rust-lang/rust/pull/76349) - + [#76667](https://github.com/rust-lang/rust/pull/76667) - + [#76708](https://github.com/rust-lang/rust/pull/76708) -- Distribute rustc sources as part of `rustc-dev` [#76856](https://github.com/rust-lang/rust/pull/76856) -- Make the default stage for x.py configurable [#76625](https://github.com/rust-lang/rust/pull/76625). This can be enabled with `build-stage = N`, `doc-stage = N`, etc. -- Add a dedicated debug-logging option [#76588](https://github.com/rust-lang/rust/pull/76588). Previously, `debug-logging` could only be set with `debug-assertions`, slowing down the compiler more than necessary. -- Add sample defaults for x.py [#76628](https://github.com/rust-lang/rust/pull/76628) -- Add `--keep-stage-std`, which behaves like `keep-stage` but allows the stage - 0 compiler artifacts (i.e., stage1/bin/rustc) to be rebuilt if changed - [#77120](https://github.com/rust-lang/rust/pull/77120). -- File locking is now used to avoid collisions between multiple running instances of `x.py` (e.g. when using `rust-analyzer` and `x.py` at the same time). Note that Solaris and possibly other non Unix and non Windows systems don't support it [#108607](https://github.com/rust-lang/rust/pull/108607). This might possibly lead to build data corruption. - - -## [Version 1] - 2020-09-11 - -This is the first changelog entry, and it does not attempt to be an exhaustive list of features in x.py. -Instead, this documents the changes to bootstrap in the past 2 months. - -- Improve defaults in `x.py` [#73964](https://github.com/rust-lang/rust/pull/73964) - (see [blog post] for details) -- Set `ninja = true` by default [#74922](https://github.com/rust-lang/rust/pull/74922) -- Avoid trying to inversely cross-compile for build triple from host triples [#76415](https://github.com/rust-lang/rust/pull/76415) -- Allow blessing expect-tests in tools [#75975](https://github.com/rust-lang/rust/pull/75975) -- `x.py check` checks tests/examples/benches [#76258](https://github.com/rust-lang/rust/pull/76258) -- Fix `rust.use-lld` when linker is not set [#76326](https://github.com/rust-lang/rust/pull/76326) -- Build tests with LLD if `use-lld = true` was passed [#76378](https://github.com/rust-lang/rust/pull/76378) - -[blog post]: https://blog.rust-lang.org/inside-rust/2020/08/30/changes-to-x-py-defaults.html diff --git a/src/bootstrap/Cargo.lock b/src/bootstrap/Cargo.lock index f5220e361..57113b0ec 100644 --- a/src/bootstrap/Cargo.lock +++ b/src/bootstrap/Cargo.lock @@ -30,6 +30,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" + +[[package]] name = "block-buffer" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -50,6 +56,7 @@ dependencies = [ "fd-lock", "filetime", "hex", + "home", "ignore", "junction", "libc", @@ -104,40 +111,38 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.2.4" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "956ac1f6381d8d82ab4684768f89c0ea3afe66925ceadb4eeb3fc452ffc55d62" +checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b" dependencies = [ "clap_builder", "clap_derive", - "once_cell", ] [[package]] name = "clap_builder" -version = "4.2.4" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84080e799e54cff944f4b4a4b0e71630b0e0443b25b985175c7dddc1a859b749" +checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663" dependencies = [ "anstyle", - "bitflags", "clap_lex", ] [[package]] name = "clap_complete" -version = "4.2.2" +version = "4.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36774babb166352bb4f7b9cb16f781ffa3439d2a8f12cd31bea85a38c888fea3" +checksum = "e3ae8ba90b9d8b007efe66e55e48fb936272f5ca00349b5b0e89877520d35ea7" dependencies = [ "clap", ] [[package]] name = "clap_derive" -version = "4.2.0" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck", "proc-macro2", @@ -147,9 +152,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.4.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" [[package]] name = "cmake" @@ -176,16 +181,6 @@ dependencies = [ ] [[package]] -name = "crossbeam-channel" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - -[[package]] name = "crossbeam-deque" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -252,30 +247,19 @@ checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" [[package]] name = "errno" -version = "0.3.0" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0" +checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" dependencies = [ - "errno-dragonfly", "libc", "windows-sys", ] [[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", - "libc", -] - -[[package]] name = "fd-lock" -version = "3.0.11" +version = "3.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9799aefb4a2e4a01cc47610b1dd47c18ab13d991f27bbcaed9296f5a53d5cbad" +checksum = "ef033ed5e9bad94e55838ca0ca906db0e043f517adda0c8b79c7a8c66c93c1b5" dependencies = [ "cfg-if", "rustix", @@ -330,27 +314,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" - -[[package]] name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] +name = "home" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "747309b4b440c06d57b0b25f2aee03ee9b5e5397d288c60e21fc709bb98a7408" +dependencies = [ + "winapi", +] + +[[package]] name = "ignore" version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -369,17 +347,6 @@ dependencies = [ ] [[package]] -name = "io-lifetimes" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" -dependencies = [ - "hermit-abi 0.3.2", - "libc", - "windows-sys", -] - -[[package]] name = "itoa" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -403,15 +370,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.140" +version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" [[package]] name = "linux-raw-sys" -version = "0.3.2" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f508063cc7bb32987c71511216bd5a32be15bccb6a80b52df8b9d7f01fc3aa2" +checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" [[package]] name = "log" @@ -458,16 +425,6 @@ dependencies = [ ] [[package]] -name = "num_cpus" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" -dependencies = [ - "hermit-abi 0.1.19", - "libc", -] - -[[package]] name = "object" version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -528,25 +485,22 @@ dependencies = [ [[package]] name = "rayon" -version = "1.6.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e060280438193c554f654141c9ea9417886713b7acd75974c85b18a69a88e0b" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" dependencies = [ - "crossbeam-deque", "either", "rayon-core", ] [[package]] name = "rayon-core" -version = "1.10.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", ] [[package]] @@ -555,7 +509,7 @@ version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -583,13 +537,12 @@ checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64" [[package]] name = "rustix" -version = "0.37.6" +version = "0.38.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d097081ed288dfe45699b72f5b5d648e5f15d64d900c7080273baa20c16a6849" +checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed" dependencies = [ - "bitflags", + "bitflags 2.4.1", "errno", - "io-lifetimes", "libc", "linux-raw-sys", "windows-sys", @@ -787,27 +740,37 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows" -version = "0.46.0" +version = "0.51.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca229916c5ee38c2f2bc1e9d8f04df975b4bd93f9955dc69fabb5d91270045c9" +dependencies = [ + "windows-core", + "windows-targets", +] + +[[package]] +name = "windows-core" +version = "0.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" dependencies = [ "windows-targets", ] [[package]] name = "windows-sys" -version = "0.45.0" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ "windows-targets", ] [[package]] name = "windows-targets" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", @@ -820,45 +783,45 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "xattr" diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index 9bf26948a..e4d359141 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -5,38 +5,48 @@ edition = "2021" build = "build.rs" default-run = "bootstrap" +[features] +build-metrics = ["sysinfo"] + [lib] -path = "lib.rs" +path = "src/lib.rs" doctest = false [[bin]] name = "bootstrap" -path = "bin/main.rs" +path = "src/bin/main.rs" test = false [[bin]] name = "rustc" -path = "bin/rustc.rs" +path = "src/bin/rustc.rs" test = false [[bin]] name = "rustdoc" -path = "bin/rustdoc.rs" +path = "src/bin/rustdoc.rs" test = false [[bin]] name = "sccache-plus-cl" -path = "bin/sccache-plus-cl.rs" +path = "src/bin/sccache-plus-cl.rs" test = false [dependencies] build_helper = { path = "../tools/build_helper" } +cc = "1.0.69" +clap = { version = "4.4.7", default-features = false, features = ["std", "usage", "help", "derive", "error-context"] } +clap_complete = "4.4.3" cmake = "0.1.38" filetime = "0.2" -cc = "1.0.69" -libc = "0.2" hex = "0.4" +home = "0.5.4" +ignore = "0.4.10" +libc = "0.2" object = { version = "0.32.0", default-features = false, features = ["archive", "coff", "read_core", "unaligned"] } +once_cell = "1.7.2" +opener = "0.5" +semver = "1.0.17" serde = "1.0.137" # Directly use serde_derive rather than through the derive feature of serde to allow building both # in parallel and to allow serde_json and toml to start building as soon as serde has been built. @@ -46,27 +56,21 @@ sha2 = "0.10" tar = "0.4" termcolor = "1.2.0" toml = "0.5" -ignore = "0.4.10" -opener = "0.5" -once_cell = "1.7.2" -xz2 = "0.1" walkdir = "2" +xz2 = "0.1" # Dependencies needed by the build-metrics feature sysinfo = { version = "0.26.0", optional = true } -clap = { version = "4.2.4", default-features = false, features = ["std", "usage", "help", "derive", "error-context"] } -clap_complete = "4.2.2" -semver = "1.0.17" # Solaris doesn't support flock() and thus fd-lock is not option now [target.'cfg(not(target_os = "solaris"))'.dependencies] -fd-lock = "3.0.8" +fd-lock = "3.0.13" [target.'cfg(windows)'.dependencies.junction] version = "1.0.0" [target.'cfg(windows)'.dependencies.windows] -version = "0.46.0" +version = "0.51.1" features = [ "Win32_Foundation", "Win32_Security", @@ -80,9 +84,6 @@ features = [ [dev-dependencies] pretty_assertions = "1.4" -[features] -build-metrics = ["sysinfo"] - # We care a lot about bootstrap's compile times, so don't include debuginfo for # dependencies, only bootstrap itself. [profile.dev] diff --git a/src/bootstrap/README.md b/src/bootstrap/README.md index 548281ca5..e7998a40a 100644 --- a/src/bootstrap/README.md +++ b/src/bootstrap/README.md @@ -181,11 +181,10 @@ Some general areas that you may be interested in modifying are: `Config` struct. * Adding a sanity check? Take a look at `bootstrap/sanity.rs`. -If you make a major change, please remember to: +If you make a major change on bootstrap configuration, please remember to: -+ Update `VERSION` in `src/bootstrap/main.rs`. -* Update `changelog-seen = N` in `config.example.toml`. -* Add an entry in `src/bootstrap/CHANGELOG.md`. ++ Update `CONFIG_CHANGE_HISTORY` in `src/bootstrap/lib.rs`. +* Update `change-id = {pull-request-id}` in `config.example.toml`. A 'major change' includes @@ -193,7 +192,7 @@ A 'major change' includes * A change in the default options. Changes that do not affect contributors to the compiler or users -building rustc from source don't need an update to `VERSION`. +building rustc from source don't need an update to `CONFIG_CHANGE_HISTORY`. If you have any questions, feel free to reach out on the `#t-infra/bootstrap` channel at [Rust Bootstrap Zulip server][rust-bootstrap-zulip]. When you encounter bugs, @@ -201,3 +200,8 @@ please file issues on the [Rust issue tracker][rust-issue-tracker]. [rust-bootstrap-zulip]: https://rust-lang.zulipchat.com/#narrow/stream/t-infra.2Fbootstrap [rust-issue-tracker]: https://github.com/rust-lang/rust/issues + +## Changelog + +Because we do not release bootstrap with versions, we also do not maintain CHANGELOG files. To +review the changes made to bootstrap, simply run `git log --no-merges --oneline -- src/bootstrap`. diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index fac0cdf20..5a84e37f8 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -211,7 +211,7 @@ def require(cmd, exit=True, exception=False): if exception: raise elif exit: - eprint("error: unable to run `{}`: {}".format(' '.join(cmd), exc)) + eprint("ERROR: unable to run `{}`: {}".format(' '.join(cmd), exc)) eprint("Please make sure it's installed and in the path.") sys.exit(1) return None @@ -681,7 +681,7 @@ class RustBuild(object): answer = self._should_fix_bins_and_dylibs = get_answer() if answer: - eprint("info: You seem to be using Nix.") + eprint("INFO: You seem to be using Nix.") return answer def fix_bin_or_dylib(self, fname): @@ -727,7 +727,7 @@ class RustBuild(object): "nix-build", "-E", nix_expr, "-o", nix_deps_dir, ]) except subprocess.CalledProcessError as reason: - eprint("warning: failed to call nix-build:", reason) + eprint("WARNING: failed to call nix-build:", reason) return self.nix_deps_dir = nix_deps_dir @@ -747,7 +747,7 @@ class RustBuild(object): try: subprocess.check_output([patchelf] + patchelf_args + [fname]) except subprocess.CalledProcessError as reason: - eprint("warning: failed to call patchelf:", reason) + eprint("WARNING: failed to call patchelf:", reason) return def rustc_stamp(self): @@ -954,6 +954,13 @@ class RustBuild(object): if deny_warnings: env["RUSTFLAGS"] += " -Dwarnings" + # Add RUSTFLAGS_BOOTSTRAP to RUSTFLAGS for bootstrap compilation. + # Note that RUSTFLAGS_BOOTSTRAP should always be added to the end of + # RUSTFLAGS to be actually effective (e.g., if we have `-Dwarnings` in + # RUSTFLAGS, passing `-Awarnings` from RUSTFLAGS_BOOTSTRAP should override it). + if "RUSTFLAGS_BOOTSTRAP" in env: + env["RUSTFLAGS"] += " " + env["RUSTFLAGS_BOOTSTRAP"] + env["PATH"] = os.path.join(self.bin_root(), "bin") + \ os.pathsep + env["PATH"] if not os.path.isfile(self.cargo()): @@ -998,7 +1005,7 @@ class RustBuild(object): if 'SUDO_USER' in os.environ and not self.use_vendored_sources: if os.getuid() == 0: self.use_vendored_sources = True - eprint('info: looks like you\'re trying to run this command as root') + eprint('INFO: looks like you\'re trying to run this command as root') eprint(' and so in order to preserve your $HOME this will now') eprint(' use vendored sources by default.') @@ -1010,14 +1017,14 @@ class RustBuild(object): "--sync ./src/tools/rust-analyzer/Cargo.toml " \ "--sync ./compiler/rustc_codegen_cranelift/Cargo.toml " \ "--sync ./src/bootstrap/Cargo.toml " - eprint('error: vendoring required, but vendor directory does not exist.') + eprint('ERROR: vendoring required, but vendor directory does not exist.') eprint(' Run `cargo vendor {}` to initialize the ' 'vendor directory.'.format(sync_dirs)) eprint('Alternatively, use the pre-vendored `rustc-src` dist component.') raise Exception("{} not found".format(vendor_dir)) if not os.path.exists(cargo_dir): - eprint('error: vendoring required, but .cargo/config does not exist.') + eprint('ERROR: vendoring required, but .cargo/config does not exist.') raise Exception("{} not found".format(cargo_dir)) else: if os.path.exists(cargo_dir): @@ -1042,6 +1049,12 @@ def bootstrap(args): """Configure, fetch, build and run the initial bootstrap""" rust_root = os.path.abspath(os.path.join(__file__, '../../..')) + if not os.path.exists(os.path.join(rust_root, '.git')) and \ + os.path.exists(os.path.join(rust_root, '.github')): + eprint("warn: Looks like you are trying to bootstrap Rust from a source that is neither a " + "git clone nor distributed tarball.\nThis build may fail due to missing submodules " + "unless you put them in place manually.") + # Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, # then `config.toml` in the root directory. toml_path = args.config or os.getenv('RUST_BOOTSTRAP_CONFIG') @@ -1112,7 +1125,7 @@ def main(): # process has to happen before anything is printed out. if help_triggered: eprint( - "info: Downloading and building bootstrap before processing --help command.\n" + "INFO: Downloading and building bootstrap before processing --help command.\n" " See src/bootstrap/README.md for help with common commands.") exit_code = 0 diff --git a/src/bootstrap/bootstrap_test.py b/src/bootstrap/bootstrap_test.py index dc06a4c97..6da410ed2 100644 --- a/src/bootstrap/bootstrap_test.py +++ b/src/bootstrap/bootstrap_test.py @@ -34,7 +34,7 @@ def serialize_and_parse(configure_args, bootstrap_args=None): # Verify this is actually valid TOML. tomllib.loads(build.config_toml) except ImportError: - print("warning: skipping TOML validation, need at least python 3.11", file=sys.stderr) + print("WARNING: skipping TOML validation, need at least python 3.11", file=sys.stderr) return build @@ -103,7 +103,6 @@ class GenerateAndParseConfig(unittest.TestCase): """Test that we can serialize and deserialize a config.toml file""" def test_no_args(self): build = serialize_and_parse([]) - self.assertEqual(build.get_toml("changelog-seen"), '2') self.assertEqual(build.get_toml("profile"), 'dist') self.assertIsNone(build.get_toml("llvm.download-ci-llvm")) diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py index f469dbea6..544a42d9a 100755 --- a/src/bootstrap/configure.py +++ b/src/bootstrap/configure.py @@ -59,6 +59,7 @@ o("missing-tools", "dist.missing-tools", "allow failures when building tools") o("use-libcxx", "llvm.use-libcxx", "build LLVM with libc++") o("control-flow-guard", "rust.control-flow-guard", "Enable Control Flow Guard") o("patch-binaries-for-nix", "build.patch-binaries-for-nix", "whether patch binaries for usage with Nix toolchains") +o("new-symbol-mangling", "rust.new-symbol-mangling", "use symbol-mangling-version v0") v("llvm-cflags", "llvm.cflags", "build LLVM with these extra compiler flags") v("llvm-cxxflags", "llvm.cxxflags", "build LLVM with these extra compiler flags") @@ -97,20 +98,7 @@ v("llvm-root", None, "set LLVM root") v("llvm-config", None, "set path to llvm-config") v("llvm-filecheck", None, "set path to LLVM's FileCheck utility") v("python", "build.python", "set path to python") -v("android-cross-path", "target.arm-linux-androideabi.android-ndk", - "Android NDK standalone path (deprecated)") -v("i686-linux-android-ndk", "target.i686-linux-android.android-ndk", - "i686-linux-android NDK standalone path") -v("arm-linux-androideabi-ndk", "target.arm-linux-androideabi.android-ndk", - "arm-linux-androideabi NDK standalone path") -v("armv7-linux-androideabi-ndk", "target.armv7-linux-androideabi.android-ndk", - "armv7-linux-androideabi NDK standalone path") -v("thumbv7neon-linux-androideabi-ndk", "target.thumbv7neon-linux-androideabi.android-ndk", - "thumbv7neon-linux-androideabi NDK standalone path") -v("aarch64-linux-android-ndk", "target.aarch64-linux-android.android-ndk", - "aarch64-linux-android NDK standalone path") -v("x86_64-linux-android-ndk", "target.x86_64-linux-android.android-ndk", - "x86_64-linux-android NDK standalone path") +v("android-ndk", "build.android-ndk", "set path to Android NDK") v("musl-root", "target.x86_64-unknown-linux-musl.musl-root", "MUSL root installation directory (deprecated)") v("musl-root-x86_64", "target.x86_64-unknown-linux-musl.musl-root", @@ -265,7 +253,7 @@ def parse_args(args): if not found: unknown_args.append(arg) - # Note: here and a few other places, we use [-1] to apply the *last* value + # NOTE: here and a few other places, we use [-1] to apply the *last* value # passed. But if option-checking is enabled, then the known_args loop will # also assert that options are only passed once. option_checking = ('option-checking' not in known_args @@ -489,7 +477,7 @@ def configure_section(lines, config): # These are used by rpm, but aren't accepted by x.py. # Give a warning that they're ignored, but not a hard error. if key in ["infodir", "localstatedir"]: - print("warning: {} will be ignored".format(key)) + print("WARNING: {} will be ignored".format(key)) else: raise RuntimeError("failed to find config line for {}".format(key)) diff --git a/src/bootstrap/defaults/config.codegen.toml b/src/bootstrap/defaults/config.codegen.toml index 113df88d7..7c33ce958 100644 --- a/src/bootstrap/defaults/config.codegen.toml +++ b/src/bootstrap/defaults/config.codegen.toml @@ -10,7 +10,7 @@ assertions = true # enable warnings during the llvm compilation enable-warnings = true # build llvm from source -download-ci-llvm = false +download-ci-llvm = "if-unchanged" [rust] # This enables `RUSTC_LOG=debug`, avoiding confusing situations diff --git a/src/bootstrap/download-ci-llvm-stamp b/src/bootstrap/download-ci-llvm-stamp index ffc380579..9998fe2f5 100644 --- a/src/bootstrap/download-ci-llvm-stamp +++ b/src/bootstrap/download-ci-llvm-stamp @@ -1,4 +1,4 @@ Change this file to make users of the `download-ci-llvm` configuration download a new version of LLVM from CI, even if the LLVM submodule hasn’t changed. -Last change is for: https://github.com/rust-lang/rust/pull/113996 +Last change is for: https://github.com/rust-lang/rust/pull/116881 diff --git a/src/bootstrap/job.rs b/src/bootstrap/job.rs deleted file mode 100644 index b0a97b540..000000000 --- a/src/bootstrap/job.rs +++ /dev/null @@ -1,143 +0,0 @@ -//! Job management on Windows for bootstrapping -//! -//! Most of the time when you're running a build system (e.g., make) you expect -//! Ctrl-C or abnormal termination to actually terminate the entire tree of -//! process in play, not just the one at the top. This currently works "by -//! default" on Unix platforms because Ctrl-C actually sends a signal to the -//! *process group* rather than the parent process, so everything will get torn -//! down. On Windows, however, this does not happen and Ctrl-C just kills the -//! parent process. -//! -//! To achieve the same semantics on Windows we use Job Objects to ensure that -//! all processes die at the same time. Job objects have a mode of operation -//! where when all handles to the object are closed it causes all child -//! processes associated with the object to be terminated immediately. -//! Conveniently whenever a process in the job object spawns a new process the -//! child will be associated with the job object as well. This means if we add -//! ourselves to the job object we create then everything will get torn down! -//! -//! Unfortunately most of the time the build system is actually called from a -//! python wrapper (which manages things like building the build system) so this -//! all doesn't quite cut it so far. To go the last mile we duplicate the job -//! object handle into our parent process (a python process probably) and then -//! close our own handle. This means that the only handle to the job object -//! resides in the parent python process, so when python dies the whole build -//! system dies (as one would probably expect!). -//! -//! Note that this module has a #[cfg(windows)] above it as none of this logic -//! is required on Unix. - -use crate::Build; -use std::env; -use std::ffi::c_void; -use std::io; -use std::mem; - -use windows::{ - core::PCWSTR, - Win32::Foundation::{CloseHandle, DuplicateHandle, DUPLICATE_SAME_ACCESS, HANDLE}, - Win32::System::Diagnostics::Debug::{SetErrorMode, SEM_NOGPFAULTERRORBOX, THREAD_ERROR_MODE}, - Win32::System::JobObjects::{ - AssignProcessToJobObject, CreateJobObjectW, JobObjectExtendedLimitInformation, - SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, - JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOB_OBJECT_LIMIT_PRIORITY_CLASS, - }, - Win32::System::Threading::{ - GetCurrentProcess, OpenProcess, BELOW_NORMAL_PRIORITY_CLASS, PROCESS_DUP_HANDLE, - }, -}; - -pub unsafe fn setup(build: &mut Build) { - // Enable the Windows Error Reporting dialog which msys disables, - // so we can JIT debug rustc - let mode = SetErrorMode(THREAD_ERROR_MODE::default()); - let mode = THREAD_ERROR_MODE(mode); - SetErrorMode(mode & !SEM_NOGPFAULTERRORBOX); - - // Create a new job object for us to use - let job = CreateJobObjectW(None, PCWSTR::null()).unwrap(); - - // Indicate that when all handles to the job object are gone that all - // process in the object should be killed. Note that this includes our - // entire process tree by default because we've added ourselves and our - // children will reside in the job by default. - let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default(); - info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; - if build.config.low_priority { - info.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_PRIORITY_CLASS; - info.BasicLimitInformation.PriorityClass = BELOW_NORMAL_PRIORITY_CLASS.0; - } - let r = SetInformationJobObject( - job, - JobObjectExtendedLimitInformation, - &info as *const _ as *const c_void, - mem::size_of_val(&info) as u32, - ) - .ok(); - assert!(r.is_ok(), "{}", io::Error::last_os_error()); - - // Assign our process to this job object. Note that if this fails, one very - // likely reason is that we are ourselves already in a job object! This can - // happen on the build bots that we've got for Windows, or if just anyone - // else is instrumenting the build. In this case we just bail out - // immediately and assume that they take care of it. - // - // Also note that nested jobs (why this might fail) are supported in recent - // versions of Windows, but the version of Windows that our bots are running - // at least don't support nested job objects. - let r = AssignProcessToJobObject(job, GetCurrentProcess()).ok(); - if r.is_err() { - CloseHandle(job); - return; - } - - // If we've got a parent process (e.g., the python script that called us) - // then move ownership of this job object up to them. That way if the python - // script is killed (e.g., via ctrl-c) then we'll all be torn down. - // - // If we don't have a parent (e.g., this was run directly) then we - // intentionally leak the job object handle. When our process exits - // (normally or abnormally) it will close the handle implicitly, causing all - // processes in the job to be cleaned up. - let pid = match env::var("BOOTSTRAP_PARENT_ID") { - Ok(s) => s, - Err(..) => return, - }; - - let parent = match OpenProcess(PROCESS_DUP_HANDLE, false, pid.parse().unwrap()).ok() { - Some(parent) => parent, - _ => { - // If we get a null parent pointer here, it is possible that either - // we have an invalid pid or the parent process has been closed. - // Since the first case rarely happens - // (only when wrongly setting the environmental variable), - // it might be better to improve the experience of the second case - // when users have interrupted the parent process and we haven't finish - // duplicating the handle yet. We just need close the job object if that occurs. - CloseHandle(job); - return; - } - }; - - let mut parent_handle = HANDLE::default(); - let r = DuplicateHandle( - GetCurrentProcess(), - job, - parent, - &mut parent_handle, - 0, - false, - DUPLICATE_SAME_ACCESS, - ) - .ok(); - - // If this failed, well at least we tried! An example of DuplicateHandle - // failing in the past has been when the wrong python2 package spawned this - // build system (e.g., the `python2` package in MSYS instead of - // `mingw-w64-x86_64-python2`). Not sure why it failed, but the "failure - // mode" here is that we only clean everything up when the build system - // dies, not when the python parent does, so not too bad. - if r.is_err() { - CloseHandle(job); - } -} diff --git a/src/bootstrap/bin/main.rs b/src/bootstrap/src/bin/main.rs index c497cabbd..0a6072ae1 100644 --- a/src/bootstrap/bin/main.rs +++ b/src/bootstrap/src/bin/main.rs @@ -13,7 +13,7 @@ use std::{env, fs}; #[cfg(all(any(unix, windows), not(target_os = "solaris")))] use bootstrap::t; -use bootstrap::{Build, Config, Subcommand, VERSION}; +use bootstrap::{find_recent_config_change_ids, Build, Config, Subcommand, CONFIG_CHANGE_HISTORY}; fn main() { let args = env::args().skip(1).collect::<Vec<_>>(); @@ -42,7 +42,7 @@ fn main() { } err => { drop(err); - println!("warning: build directory locked by process {pid}, waiting for lock"); + println!("WARNING: build directory locked by process {pid}, waiting for lock"); let mut lock = t!(build_lock.write()); t!(lock.write(&process::id().to_string().as_ref())); lock @@ -51,7 +51,7 @@ fn main() { } #[cfg(any(not(any(unix, windows)), target_os = "solaris"))] - println!("warning: file locking not supported for target, not locking build directory"); + println!("WARNING: file locking not supported for target, not locking build directory"); // check_version warnings are not printed during setup let changelog_suggestion = @@ -61,9 +61,9 @@ fn main() { // changelog warning, not the `x.py setup` message. let suggest_setup = config.config.is_none() && !matches!(config.cmd, Subcommand::Setup { .. }); if suggest_setup { - println!("warning: you have not made a `config.toml`"); + println!("WARNING: you have not made a `config.toml`"); println!( - "help: consider running `./x.py setup` or copying `config.example.toml` by running \ + "HELP: consider running `./x.py setup` or copying `config.example.toml` by running \ `cp config.example.toml config.toml`" ); } else if let Some(suggestion) = &changelog_suggestion { @@ -74,9 +74,9 @@ fn main() { Build::new(config).build(); if suggest_setup { - println!("warning: you have not made a `config.toml`"); + println!("WARNING: you have not made a `config.toml`"); println!( - "help: consider running `./x.py setup` or copying `config.example.toml` by running \ + "HELP: consider running `./x.py setup` or copying `config.example.toml` by running \ `cp config.example.toml config.toml`" ); } else if let Some(suggestion) = &changelog_suggestion { @@ -91,34 +91,53 @@ fn main() { contents.contains("https://github.com/rust-lang/rust/issues/77620#issuecomment-705144570") }) { println!( - "warning: You have the pre-push script installed to .git/hooks/pre-commit. \ + "WARNING: You have the pre-push script installed to .git/hooks/pre-commit. \ Consider moving it to .git/hooks/pre-push instead, which runs less often." ); } if suggest_setup || changelog_suggestion.is_some() { - println!("note: this message was printed twice to make it more likely to be seen"); + println!("NOTE: this message was printed twice to make it more likely to be seen"); } } fn check_version(config: &Config) -> Option<String> { let mut msg = String::new(); - let suggestion = if let Some(seen) = config.changelog_seen { - if seen != VERSION { - msg.push_str("warning: there have been changes to x.py since you last updated.\n"); - format!("update `config.toml` to use `changelog-seen = {VERSION}` instead") - } else { + if config.changelog_seen.is_some() { + msg.push_str("WARNING: The use of `changelog-seen` is deprecated. Please refer to `change-id` option in `config.example.toml` instead.\n"); + } + + let latest_config_id = CONFIG_CHANGE_HISTORY.last().unwrap(); + if let Some(id) = config.change_id { + if &id == latest_config_id { return None; } + + let change_links: Vec<String> = find_recent_config_change_ids(id) + .iter() + .map(|id| format!("https://github.com/rust-lang/rust/pull/{id}")) + .collect(); + if !change_links.is_empty() { + msg.push_str("WARNING: there have been changes to x.py since you last updated.\n"); + msg.push_str("To see more detail about these changes, visit the following PRs:\n"); + + for link in change_links { + msg.push_str(&format!(" - {link}\n")); + } + + msg.push_str("WARNING: there have been changes to x.py since you last updated.\n"); + + msg.push_str("NOTE: to silence this warning, "); + msg.push_str(&format!( + "update `config.toml` to use `change-id = {latest_config_id}` instead" + )); + } } else { - msg.push_str("warning: x.py has made several changes recently you may want to look at\n"); - format!("add `changelog-seen = {VERSION}` at the top of `config.toml`") + msg.push_str("WARNING: The `change-id` is missing in the `config.toml`. This means that you will not be able to track the major changes made to the bootstrap configurations.\n"); + msg.push_str("NOTE: to silence this warning, "); + msg.push_str(&format!("add `change-id = {latest_config_id}` at the top of `config.toml`")); }; - msg.push_str("help: consider looking at the changes in `src/bootstrap/CHANGELOG.md`\n"); - msg.push_str("note: to silence this warning, "); - msg.push_str(&suggestion); - Some(msg) } diff --git a/src/bootstrap/bin/rustc.rs b/src/bootstrap/src/bin/rustc.rs index 20cd63b96..070a2da6a 100644 --- a/src/bootstrap/bin/rustc.rs +++ b/src/bootstrap/src/bin/rustc.rs @@ -15,20 +15,27 @@ //! switching compilers for the bootstrap and for build scripts will probably //! never get replaced. -include!("../dylib_util.rs"); -include!("./_helper.rs"); - use std::env; use std::path::PathBuf; -use std::process::{exit, Child, Command}; +use std::process::{Child, Command}; use std::time::Instant; +use dylib_util::{dylib_path, dylib_path_var}; + +#[path = "../utils/bin_helpers.rs"] +mod bin_helpers; + +#[path = "../utils/dylib.rs"] +mod dylib_util; + fn main() { let args = env::args_os().skip(1).collect::<Vec<_>>(); let arg = |name| args.windows(2).find(|args| args[0] == name).and_then(|args| args[1].to_str()); - let stage = parse_rustc_stage(); - let verbose = parse_rustc_verbose(); + // We don't use the stage in this shim, but let's parse it to make sure that we're invoked + // by bootstrap, or that we provide a helpful error message if not. + bin_helpers::parse_rustc_stage(); + let verbose = bin_helpers::parse_rustc_verbose(); // Detect whether or not we're a build script depending on whether --target // is passed (a bit janky...) @@ -108,41 +115,25 @@ fn main() { cmd.arg("-Ztls-model=initial-exec"); } } else { - // FIXME(rust-lang/cargo#5754) we shouldn't be using special env vars - // here, but rather Cargo should know what flags to pass rustc itself. - - // Override linker if necessary. - if let Ok(host_linker) = env::var("RUSTC_HOST_LINKER") { - cmd.arg(format!("-Clinker={host_linker}")); - } - if env::var_os("RUSTC_HOST_FUSE_LD_LLD").is_some() { - cmd.arg("-Clink-args=-fuse-ld=lld"); - } - - if let Ok(s) = env::var("RUSTC_HOST_CRT_STATIC") { - if s == "true" { - cmd.arg("-C").arg("target-feature=+crt-static"); - } - if s == "false" { - cmd.arg("-C").arg("target-feature=-crt-static"); + // Find any host flags that were passed by bootstrap. + // The flags are stored in a RUSTC_HOST_FLAGS variable, separated by spaces. + if let Ok(flags) = std::env::var("RUSTC_HOST_FLAGS") { + for flag in flags.split(' ') { + cmd.arg(flag); } } - - // Cargo doesn't pass RUSTFLAGS to proc_macros: - // https://github.com/rust-lang/cargo/issues/4423 - // Thus, if we are on stage 0, we explicitly set `--cfg=bootstrap`. - // We also declare that the flag is expected, which we need to do to not - // get warnings about it being unexpected. - if stage == "0" { - cmd.arg("--cfg=bootstrap"); - } - cmd.arg("-Zunstable-options"); - cmd.arg("--check-cfg=values(bootstrap)"); } if let Ok(map) = env::var("RUSTC_DEBUGINFO_MAP") { cmd.arg("--remap-path-prefix").arg(&map); } + // The remap flags for Cargo registry sources need to be passed after the remapping for the + // Rust source code directory, to handle cases when $CARGO_HOME is inside the source directory. + if let Ok(maps) = env::var("RUSTC_CARGO_REGISTRY_SRC_TO_REMAP") { + for map in maps.split('\t') { + cmd.arg("--remap-path-prefix").arg(map); + } + } // Force all crates compiled by this compiler to (a) be unstable and (b) // allow the `rustc_private` feature to link to other unstable crates @@ -217,6 +208,12 @@ fn main() { eprintln!("{prefix} libdir: {libdir:?}"); } + if env::var_os("RUSTC_BOLT_LINK_FLAGS").is_some() { + if let Some("rustc_driver") = crate_name { + cmd.arg("-Clink-args=-Wl,-q"); + } + } + let start = Instant::now(); let (child, status) = { let errmsg = format!("\nFailed to run:\n{cmd:?}\n-------------"); @@ -248,7 +245,7 @@ fn main() { if status.success() { std::process::exit(0); - // note: everything below here is unreachable. do not put code that + // NOTE: everything below here is unreachable. do not put code that // should run on success, after this block. } if verbose > 0 { @@ -306,10 +303,9 @@ fn format_rusage_data(child: Child) -> Option<String> { &mut user_filetime, ) } - .ok() .ok()?; - unsafe { FileTimeToSystemTime(&user_filetime, &mut user_time) }.ok().ok()?; - unsafe { FileTimeToSystemTime(&kernel_filetime, &mut kernel_time) }.ok().ok()?; + unsafe { FileTimeToSystemTime(&user_filetime, &mut user_time) }.ok()?; + unsafe { FileTimeToSystemTime(&kernel_filetime, &mut kernel_time) }.ok()?; // Unlike on Linux with RUSAGE_CHILDREN, this will only return memory information for the process // with the given handle and none of that process's children. diff --git a/src/bootstrap/bin/rustdoc.rs b/src/bootstrap/src/bin/rustdoc.rs index 6561c1c19..dbbce6fe2 100644 --- a/src/bootstrap/bin/rustdoc.rs +++ b/src/bootstrap/src/bin/rustdoc.rs @@ -5,17 +5,21 @@ use std::env; use std::ffi::OsString; use std::path::PathBuf; -use std::process::{exit, Command}; +use std::process::Command; -include!("../dylib_util.rs"); +use dylib_util::{dylib_path, dylib_path_var}; -include!("./_helper.rs"); +#[path = "../utils/bin_helpers.rs"] +mod bin_helpers; + +#[path = "../utils/dylib.rs"] +mod dylib_util; fn main() { let args = env::args_os().skip(1).collect::<Vec<_>>(); - let stage = parse_rustc_stage(); - let verbose = parse_rustc_verbose(); + let stage = bin_helpers::parse_rustc_stage(); + let verbose = bin_helpers::parse_rustc_verbose(); let rustdoc = env::var_os("RUSTDOC_REAL").expect("RUSTDOC_REAL was not set"); let libdir = env::var_os("RUSTDOC_LIBDIR").expect("RUSTDOC_LIBDIR was not set"); @@ -66,7 +70,9 @@ fn main() { cmd.arg("--cfg=bootstrap"); } cmd.arg("-Zunstable-options"); + // #[cfg(bootstrap)] cmd.arg("--check-cfg=values(bootstrap)"); + // cmd.arg("--check-cfg=cfg(bootstrap)"); if verbose > 1 { eprintln!( diff --git a/src/bootstrap/bin/sccache-plus-cl.rs b/src/bootstrap/src/bin/sccache-plus-cl.rs index 554c2dd4d..554c2dd4d 100644 --- a/src/bootstrap/bin/sccache-plus-cl.rs +++ b/src/bootstrap/src/bin/sccache-plus-cl.rs diff --git a/src/bootstrap/check.rs b/src/bootstrap/src/core/build_steps/check.rs index b417abc00..121925b56 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/src/core/build_steps/check.rs @@ -1,10 +1,12 @@ //! Implementation of compiling the compiler and standard library, in "check"-based modes. -use crate::builder::{crate_description, Alias, Builder, Kind, RunConfig, ShouldRun, Step}; -use crate::cache::Interned; -use crate::compile::{add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo}; -use crate::config::TargetSelection; -use crate::tool::{prepare_tool_cargo, SourceType}; +use crate::core::build_steps::compile::{ + add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo, +}; +use crate::core::build_steps::tool::{prepare_tool_cargo, SourceType}; +use crate::core::builder::{crate_description, Alias, Builder, Kind, RunConfig, ShouldRun, Step}; +use crate::core::config::TargetSelection; +use crate::utils::cache::Interned; use crate::INTERNER; use crate::{Compiler, Mode, Subcommand}; use std::path::{Path, PathBuf}; @@ -16,7 +18,7 @@ pub struct Std { /// /// This shouldn't be used from other steps; see the comment on [`compile::Rustc`]. /// - /// [`compile::Rustc`]: crate::compile::Rustc + /// [`compile::Rustc`]: crate::core::build_steps::compile::Rustc crates: Interned<Vec<String>>, } @@ -193,7 +195,7 @@ pub struct Rustc { /// /// This shouldn't be used from other steps; see the comment on [`compile::Rustc`]. /// - /// [`compile::Rustc`]: crate::compile::Rustc + /// [`compile::Rustc`]: crate::core::build_steps::compile::Rustc crates: Interned<Vec<String>>, } @@ -237,8 +239,8 @@ impl Step for Rustc { // the sysroot for the compiler to find. Otherwise, we're going to // fail when building crates that need to generate code (e.g., build // scripts and their dependencies). - builder.ensure(crate::compile::Std::new(compiler, compiler.host)); - builder.ensure(crate::compile::Std::new(compiler, target)); + builder.ensure(crate::core::build_steps::compile::Std::new(compiler, compiler.host)); + builder.ensure(crate::core::build_steps::compile::Std::new(compiler, target)); } else { builder.ensure(Std::new(target)); } @@ -387,7 +389,7 @@ impl Step for RustAnalyzer { &["rust-analyzer/in-rust-tree".to_owned()], ); - cargo.allow_features(crate::tool::RustAnalyzer::ALLOW_FEATURES); + cargo.allow_features(crate::core::build_steps::tool::RustAnalyzer::ALLOW_FEATURES); // For ./x.py clippy, don't check those targets because // linting tests and benchmarks can produce very noisy results diff --git a/src/bootstrap/clean.rs b/src/bootstrap/src/core/build_steps/clean.rs index 7389816b4..cbb6b5f46 100644 --- a/src/bootstrap/clean.rs +++ b/src/bootstrap/src/core/build_steps/clean.rs @@ -9,9 +9,9 @@ use std::fs; use std::io::{self, ErrorKind}; use std::path::Path; -use crate::builder::{crate_description, Builder, RunConfig, ShouldRun, Step}; -use crate::cache::Interned; -use crate::util::t; +use crate::core::builder::{crate_description, Builder, RunConfig, ShouldRun, Step}; +use crate::utils::cache::Interned; +use crate::utils::helpers::t; use crate::{Build, Compiler, Mode, Subcommand}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -68,6 +68,12 @@ macro_rules! clean_crate_tree { let compiler = self.compiler; let target = compiler.host; let mut cargo = builder.bare_cargo(compiler, $mode, target, "clean"); + + // Since https://github.com/rust-lang/rust/pull/111076 enables + // unstable cargo feature (`public-dependency`), we need to ensure + // that unstable features are enabled before reading libstd Cargo.toml. + cargo.env("RUSTC_BOOTSTRAP", "1"); + for krate in &*self.crates { cargo.arg("-p"); cargo.arg(krate); @@ -139,7 +145,6 @@ fn clean_specific_stage(build: &Build, stage: u32) { fn clean_default(build: &Build) { rm_rf(&build.out.join("tmp")); rm_rf(&build.out.join("dist")); - rm_rf(&build.out.join("bootstrap")); rm_rf(&build.out.join("rustfmt.stamp")); for host in &build.hosts { @@ -178,7 +183,7 @@ fn rm_rf(path: &Path) { && p.file_name().and_then(std::ffi::OsStr::to_str) == Some("bootstrap.exe") { - eprintln!("warning: failed to delete '{}'.", p.display()); + eprintln!("WARNING: failed to delete '{}'.", p.display()); return Ok(()); } Err(e) diff --git a/src/bootstrap/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 292ccc578..7021a9543 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -19,16 +19,17 @@ use std::str; use serde_derive::Deserialize; -use crate::builder::crate_description; -use crate::builder::Cargo; -use crate::builder::{Builder, Kind, PathSet, RunConfig, ShouldRun, Step, TaskPath}; -use crate::cache::{Interned, INTERNER}; -use crate::config::{DebuginfoLevel, LlvmLibunwind, RustcLto, TargetSelection}; -use crate::dist; -use crate::llvm; -use crate::tool::SourceType; -use crate::util::get_clang_cl_resource_dir; -use crate::util::{exe, is_debug_info, is_dylib, output, symlink_dir, t, up_to_date}; +use crate::core::build_steps::dist; +use crate::core::build_steps::llvm; +use crate::core::build_steps::tool::SourceType; +use crate::core::builder::crate_description; +use crate::core::builder::Cargo; +use crate::core::builder::{Builder, Kind, PathSet, RunConfig, ShouldRun, Step, TaskPath}; +use crate::core::config::{DebuginfoLevel, LlvmLibunwind, RustcLto, TargetSelection}; +use crate::utils::cache::{Interned, INTERNER}; +use crate::utils::helpers::{ + exe, get_clang_cl_resource_dir, is_debug_info, is_dylib, output, symlink_dir, t, up_to_date, +}; use crate::LLVM_TOOLS; use crate::{CLang, Compiler, DependencyType, GitRepo, Mode}; use filetime::FileTime; @@ -44,15 +45,42 @@ pub struct Std { /// When using download-rustc, we need to use a new build of `std` for running unit tests of Std itself, /// but we need to use the downloaded copy of std for linking to rustdoc. Allow this to be overriden by `builder.ensure` from other steps. force_recompile: bool, + extra_rust_args: &'static [&'static str], } impl Std { pub fn new(compiler: Compiler, target: TargetSelection) -> Self { - Self { target, compiler, crates: Default::default(), force_recompile: false } + Self { + target, + compiler, + crates: Default::default(), + force_recompile: false, + extra_rust_args: &[], + } } pub fn force_recompile(compiler: Compiler, target: TargetSelection) -> Self { - Self { target, compiler, crates: Default::default(), force_recompile: true } + Self { + target, + compiler, + crates: Default::default(), + force_recompile: true, + extra_rust_args: &[], + } + } + + pub fn new_with_extra_rust_args( + compiler: Compiler, + target: TargetSelection, + extra_rust_args: &'static [&'static str], + ) -> Self { + Self { + target, + compiler, + crates: Default::default(), + force_recompile: false, + extra_rust_args, + } } } @@ -80,6 +108,7 @@ impl Step for Std { target: run.target, crates, force_recompile: false, + extra_rust_args: &[], }); } @@ -112,7 +141,7 @@ impl Step for Std { if builder.config.keep_stage.contains(&compiler.stage) || builder.config.keep_stage_std.contains(&compiler.stage) { - builder.info("Warning: Using a potentially old libstd. This may not behave well."); + builder.info("WARNING: Using a potentially old libstd. This may not behave well."); copy_third_party_objects(builder, &compiler, target); copy_self_contained_objects(builder, &compiler, target); @@ -158,6 +187,25 @@ impl Step for Std { target_deps.extend(copy_third_party_objects(builder, &compiler, target)); target_deps.extend(copy_self_contained_objects(builder, &compiler, target)); + // The LLD wrappers and `rust-lld` are self-contained linking components that can be + // necessary to link the stdlib on some targets. We'll also need to copy these binaries to + // the `stage0-sysroot` to ensure the linker is found when bootstrapping on such a target. + if compiler.stage == 0 && compiler.host == builder.config.build { + // We want to copy the host `bin` folder within the `rustlib` folder in the sysroot. + let src_sysroot_bin = builder + .rustc_snapshot_sysroot() + .join("lib") + .join("rustlib") + .join(compiler.host.triple) + .join("bin"); + if src_sysroot_bin.exists() { + let target_sysroot_bin = + builder.sysroot_libdir(compiler, target).parent().unwrap().join("bin"); + t!(fs::create_dir_all(&target_sysroot_bin)); + builder.cp_r(&src_sysroot_bin, &target_sysroot_bin); + } + } + let mut cargo = builder.cargo(compiler, Mode::Std, SourceType::InTree, target, "build"); std_cargo(builder, target, compiler.stage, &mut cargo); for krate in &*self.crates { @@ -168,6 +216,9 @@ impl Step for Std { if target.is_synthetic() { cargo.env("RUSTC_BOOTSTRAP_SYNTHETIC_TARGET", "1"); } + for rustflag in self.extra_rust_args.into_iter() { + cargo.rustflag(rustflag); + } let _guard = builder.msg( Kind::Build, @@ -362,11 +413,6 @@ pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, car let mut features = String::new(); - // Cranelift doesn't support `asm`. - if stage != 0 && builder.config.default_codegen_backend().unwrap_or_default() == "cranelift" { - features += " compiler-builtins-no-asm"; - } - if builder.no_std(target) == Some(true) { features += " compiler-builtins-mem"; if !target.starts_with("bpf") { @@ -491,7 +537,7 @@ impl Step for StdLink { let (libdir, hostdir) = if self.force_recompile && builder.download_rustc() { // NOTE: copies part of `sysroot_libdir` to avoid having to add a new `force_recompile` argument there too let lib = builder.sysroot_libdir_relative(self.compiler); - let sysroot = builder.ensure(crate::compile::Sysroot { + let sysroot = builder.ensure(crate::core::build_steps::compile::Sysroot { compiler: self.compiler, force_recompile: self.force_recompile, }); @@ -771,8 +817,8 @@ impl Step for Rustc { builder.ensure(Std::new(compiler, target)); if builder.config.keep_stage.contains(&compiler.stage) { - builder.info("Warning: Using a potentially old librustc. This may not behave well."); - builder.info("Warning: Use `--keep-stage-std` if you want to rebuild the compiler when it changes"); + builder.info("WARNING: Using a potentially old librustc. This may not behave well."); + builder.info("WARNING: Use `--keep-stage-std` if you want to rebuild the compiler when it changes"); builder.ensure(RustcLink::from_rustc(self, compiler)); return; } @@ -887,6 +933,11 @@ impl Step for Rustc { cargo.arg("-p").arg(krate); } + if builder.build.config.enable_bolt_settings && compiler.stage == 1 { + // Relocations are required for BOLT to work. + cargo.env("RUSTC_BOLT_LINK_FLAGS", "1"); + } + let _guard = builder.msg_sysroot_tool( Kind::Build, compiler.stage, @@ -992,7 +1043,8 @@ pub fn rustc_cargo_env( // detected that LLVM is already built and good to go which helps prevent // busting caches (e.g. like #71152). if builder.config.llvm_enabled() { - let building_is_expensive = crate::llvm::prebuilt_llvm_config(builder, target).is_err(); + let building_is_expensive = + crate::core::build_steps::llvm::prebuilt_llvm_config(builder, target).is_err(); // `top_stage == stage` might be false for `check --stage 1`, if we are building the stage 1 compiler let can_skip_build = builder.kind == Kind::Check && builder.top_stage == stage; let should_skip_build = building_is_expensive && can_skip_build; @@ -1151,8 +1203,8 @@ fn is_codegen_cfg_needed(path: &TaskPath, run: &RunConfig<'_>) -> bool { } if needs_codegen_backend_config { run.builder.info( - "Warning: no codegen-backends config matched the requested path to build a codegen backend. \ - Help: add backend to codegen-backends in config.toml.", + "WARNING: no codegen-backends config matched the requested path to build a codegen backend. \ + HELP: add backend to codegen-backends in config.toml.", ); return true; } @@ -1198,7 +1250,7 @@ impl Step for CodegenBackend { if builder.config.keep_stage.contains(&compiler.stage) { builder.info( - "Warning: Using a potentially old codegen backend. \ + "WARNING: Using a potentially old codegen backend. \ This may not behave well.", ); // Codegen backends are linked separately from this step today, so we don't do @@ -1473,14 +1525,14 @@ impl Step for Sysroot { let sysroot_lib_rustlib_src_rust = sysroot_lib_rustlib_src.join("rust"); if let Err(e) = symlink_dir(&builder.config, &builder.src, &sysroot_lib_rustlib_src_rust) { eprintln!( - "warning: creating symbolic link `{}` to `{}` failed with {}", + "WARNING: creating symbolic link `{}` to `{}` failed with {}", sysroot_lib_rustlib_src_rust.display(), builder.src.display(), e, ); if builder.config.rust_remap_debuginfo { eprintln!( - "warning: some `tests/ui` tests will fail when lacking `{}`", + "WARNING: some `tests/ui` tests will fail when lacking `{}`", sysroot_lib_rustlib_src_rust.display(), ); } @@ -1493,7 +1545,7 @@ impl Step for Sysroot { symlink_dir(&builder.config, &builder.src, &sysroot_lib_rustlib_rustcsrc_rust) { eprintln!( - "warning: creating symbolic link `{}` to `{}` failed with {}", + "WARNING: creating symbolic link `{}` to `{}` failed with {}", sysroot_lib_rustlib_rustcsrc_rust.display(), builder.src.display(), e, @@ -1658,15 +1710,17 @@ impl Step for Assemble { let src_exe = exe("lld", target_compiler.host); let dst_exe = exe("rust-lld", target_compiler.host); builder.copy(&lld_install.join("bin").join(&src_exe), &libdir_bin.join(&dst_exe)); - // for `-Z gcc-ld=lld` - let gcc_ld_dir = libdir_bin.join("gcc-ld"); - t!(fs::create_dir(&gcc_ld_dir)); - let lld_wrapper_exe = builder.ensure(crate::tool::LldWrapper { + let self_contained_lld_dir = libdir_bin.join("gcc-ld"); + t!(fs::create_dir(&self_contained_lld_dir)); + let lld_wrapper_exe = builder.ensure(crate::core::build_steps::tool::LldWrapper { compiler: build_compiler, target: target_compiler.host, }); for name in crate::LLD_FILE_NAMES { - builder.copy(&lld_wrapper_exe, &gcc_ld_dir.join(exe(name, target_compiler.host))); + builder.copy( + &lld_wrapper_exe, + &self_contained_lld_dir.join(exe(name, target_compiler.host)), + ); } } @@ -1932,7 +1986,7 @@ pub fn stream_cargo( builder.verbose(&format!("running: {cargo:?}")); let mut child = match cargo.spawn() { Ok(child) => child, - Err(e) => panic!("failed to execute command: {cargo:?}\nerror: {e}"), + Err(e) => panic!("failed to execute command: {cargo:?}\nERROR: {e}"), }; // Spawn Cargo slurping up its JSON output. We'll start building up the @@ -1996,7 +2050,7 @@ pub fn strip_debug(builder: &Builder<'_>, target: TargetSelection, path: &Path) } let previous_mtime = FileTime::from_last_modification_time(&path.metadata().unwrap()); - // Note: `output` will propagate any errors here. + // NOTE: `output` will propagate any errors here. output(Command::new("strip").arg("--strip-debug").arg(path)); // After running `strip`, we have to set the file modification time to what it was before, diff --git a/src/bootstrap/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 32da4ac29..c485481b9 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -19,16 +19,16 @@ use std::process::Command; use object::read::archive::ArchiveFile; use object::BinaryFormat; -use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; -use crate::cache::{Interned, INTERNER}; -use crate::channel; -use crate::compile; -use crate::config::TargetSelection; -use crate::doc::DocumentationFormat; -use crate::llvm; -use crate::tarball::{GeneratedTarball, OverlayKind, Tarball}; -use crate::tool::{self, Tool}; -use crate::util::{exe, is_dylib, output, t, timeit}; +use crate::core::build_steps::compile; +use crate::core::build_steps::doc::DocumentationFormat; +use crate::core::build_steps::llvm; +use crate::core::build_steps::tool::{self, Tool}; +use crate::core::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; +use crate::core::config::TargetSelection; +use crate::utils::cache::{Interned, INTERNER}; +use crate::utils::channel; +use crate::utils::helpers::{exe, is_dylib, output, t, target_supports_cranelift_backend, timeit}; +use crate::utils::tarball::{GeneratedTarball, OverlayKind, Tarball}; use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS}; pub fn pkgname(builder: &Builder<'_>, component: &str) -> String { @@ -104,7 +104,7 @@ impl Step for JsonDocs { /// Builds the `rust-docs-json` installer component. fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { let host = self.host; - builder.ensure(crate::doc::Std::new( + builder.ensure(crate::core::build_steps::doc::Std::new( builder.top_stage, host, builder, @@ -443,19 +443,6 @@ impl Step for Rustc { } } - // Copy over the codegen backends - let backends_src = builder.sysroot_codegen_backends(compiler); - let backends_rel = backends_src - .strip_prefix(&src) - .unwrap() - .strip_prefix(builder.sysroot_libdir_relative(compiler)) - .unwrap(); - // Don't use custom libdir here because ^lib/ will be resolved again with installer - let backends_dst = image.join("lib").join(&backends_rel); - - t!(fs::create_dir_all(&backends_dst)); - builder.cp_r(&backends_src, &backends_dst); - // Copy libLLVM.so to the lib dir as well, if needed. While not // technically needed by rustc itself it's needed by lots of other // components like the llvm tools and LLD. LLD is included below and @@ -471,14 +458,15 @@ impl Step for Rustc { let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin"); let rust_lld = exe("rust-lld", compiler.host); builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld)); - // for `-Z gcc-ld=lld` - let gcc_lld_src_dir = src_dir.join("gcc-ld"); - let gcc_lld_dst_dir = dst_dir.join("gcc-ld"); - t!(fs::create_dir(&gcc_lld_dst_dir)); + let self_contained_lld_src_dir = src_dir.join("gcc-ld"); + let self_contained_lld_dst_dir = dst_dir.join("gcc-ld"); + t!(fs::create_dir(&self_contained_lld_dst_dir)); for name in crate::LLD_FILE_NAMES { let exe_name = exe(name, compiler.host); - builder - .copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name)); + builder.copy( + &self_contained_lld_src_dir.join(&exe_name), + &self_contained_lld_dst_dir.join(&exe_name), + ); } } @@ -487,7 +475,7 @@ impl Step for Rustc { let man_src = builder.src.join("src/doc/man"); let man_dst = image.join("share/man/man1"); - // don't use our `bootstrap::util::{copy, cp_r}`, because those try + // don't use our `bootstrap::{copy, cp_r}`, because those try // to hardlink, and we don't want to edit the source templates for file_entry in builder.read_dir(&man_src) { let page_src = file_entry.path(); @@ -1001,11 +989,15 @@ impl Step for PlainSourceTarball { channel::write_commit_info_file(&plain_dst_src, info); } - // If we're building from git sources, we need to vendor a complete distribution. - if builder.rust_info().is_managed_git_subrepository() { - // Ensure we have the submodules checked out. - builder.update_submodule(Path::new("src/tools/cargo")); - builder.update_submodule(Path::new("src/tools/rust-analyzer")); + // If we're building from git or tarball sources, we need to vendor + // a complete distribution. + if builder.rust_info().is_managed_git_subrepository() + || builder.rust_info().is_from_tarball() + { + if builder.rust_info().is_managed_git_subrepository() { + // Ensure we have the submodules checked out. + builder.update_submodule(Path::new("src/tools/cargo")); + } // Vendor all Cargo dependencies let mut cmd = Command::new(&builder.initial_cargo); @@ -1278,6 +1270,102 @@ impl Step for Miri { } #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct CodegenBackend { + pub compiler: Compiler, + pub backend: Interned<String>, +} + +impl Step for CodegenBackend { + type Output = Option<GeneratedTarball>; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("compiler/rustc_codegen_cranelift") + } + + fn make_run(run: RunConfig<'_>) { + for &backend in &run.builder.config.rust_codegen_backends { + if backend == "llvm" { + continue; // Already built as part of rustc + } + + run.builder.ensure(CodegenBackend { + compiler: run.builder.compiler(run.builder.top_stage, run.target), + backend, + }); + } + } + + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { + if builder.config.dry_run() { + return None; + } + + // This prevents rustc_codegen_cranelift from being built for "dist" + // or "install" on the stable/beta channels. It is not yet stable and + // should not be included. + if !builder.build.unstable_features() { + return None; + } + + if !builder.config.rust_codegen_backends.contains(&self.backend) { + return None; + } + + if self.backend == "cranelift" { + if !target_supports_cranelift_backend(self.compiler.host) { + builder.info("target not supported by rustc_codegen_cranelift. skipping"); + return None; + } + + if self.compiler.host.contains("windows") { + builder.info( + "dist currently disabled for windows by rustc_codegen_cranelift. skipping", + ); + return None; + } + } + + let compiler = self.compiler; + let backend = self.backend; + + let mut tarball = + Tarball::new(builder, &format!("rustc-codegen-{}", backend), &compiler.host.triple); + if backend == "cranelift" { + tarball.set_overlay(OverlayKind::RustcCodegenCranelift); + } else { + panic!("Unknown backend rustc_codegen_{}", backend); + } + tarball.is_preview(true); + tarball.add_legal_and_readme_to(format!("share/doc/rustc_codegen_{}", backend)); + + let src = builder.sysroot(compiler); + let backends_src = builder.sysroot_codegen_backends(compiler); + let backends_rel = backends_src + .strip_prefix(&src) + .unwrap() + .strip_prefix(builder.sysroot_libdir_relative(compiler)) + .unwrap(); + // Don't use custom libdir here because ^lib/ will be resolved again with installer + let backends_dst = PathBuf::from("lib").join(&backends_rel); + + let backend_name = format!("rustc_codegen_{}", backend); + let mut found_backend = false; + for backend in fs::read_dir(&backends_src).unwrap() { + let file_name = backend.unwrap().file_name(); + if file_name.to_str().unwrap().contains(&backend_name) { + tarball.add_file(backends_src.join(file_name), &backends_dst, 0o644); + found_backend = true; + } + } + assert!(found_backend); + + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] pub struct Rustfmt { pub compiler: Compiler, pub target: TargetSelection, @@ -1447,6 +1535,10 @@ impl Step for Extended { add_component!("clippy" => Clippy { compiler, target }); add_component!("miri" => Miri { compiler, target }); add_component!("analysis" => Analysis { compiler, target }); + add_component!("rustc-codegen-cranelift" => CodegenBackend { + compiler: builder.compiler(stage, target), + backend: INTERNER.intern_str("cranelift"), + }); let etc = builder.src.join("src/etc/installer"); @@ -1538,7 +1630,7 @@ impl Step for Extended { prepare("rust-analysis"); prepare("clippy"); prepare("rust-analyzer"); - for tool in &["rust-docs", "rust-demangler", "miri"] { + for tool in &["rust-docs", "rust-demangler", "miri", "rustc-codegen-cranelift"] { if built_tools.contains(tool) { prepare(tool); } @@ -1582,6 +1674,10 @@ impl Step for Extended { "rust-demangler-preview".to_string() } else if name == "miri" { "miri-preview".to_string() + } else if name == "rustc-codegen-cranelift" { + // FIXME add installer support for cg_clif once it is ready to be distributed on + // windows. + unreachable!("cg_clif shouldn't be built for windows"); } else { name.to_string() }; @@ -2055,7 +2151,7 @@ impl Step for LlvmTools { } } - builder.ensure(crate::llvm::Llvm { target }); + builder.ensure(crate::core::build_steps::llvm::Llvm { target }); let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple); tarball.set_overlay(OverlayKind::LLVM); @@ -2114,29 +2210,25 @@ impl Step for RustDev { let mut tarball = Tarball::new(builder, "rust-dev", &target.triple); tarball.set_overlay(OverlayKind::LLVM); - builder.ensure(crate::llvm::Llvm { target }); + builder.ensure(crate::core::build_steps::llvm::Llvm { target }); // We want to package `lld` to use it with `download-ci-llvm`. - builder.ensure(crate::llvm::Lld { target }); + builder.ensure(crate::core::build_steps::llvm::Lld { target }); let src_bindir = builder.llvm_out(target).join("bin"); - // If updating this list, you likely want to change + // If updating this, you likely want to change // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users // will not pick up the extra file until LLVM gets bumped. - for bin in &[ - "llvm-config", - "llvm-ar", - "llvm-objdump", - "llvm-profdata", - "llvm-bcanalyzer", - "llvm-cov", - "llvm-dwp", - "llvm-nm", - "llvm-dwarfdump", - "llvm-dis", - "llvm-tblgen", - ] { - tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755); + // We should include all the build artifacts obtained from a source build, + // so that you can use the downloadable LLVM as if you’ve just run a full source build. + if src_bindir.exists() { + for entry in walkdir::WalkDir::new(&src_bindir) { + let entry = t!(entry); + if entry.file_type().is_file() && !entry.path_is_symlink() { + let name = entry.file_name().to_str().unwrap(); + tarball.add_file(src_bindir.join(name), "bin", 0o755); + } + } } // We don't build LLD on some platforms, so only add it if it exists diff --git a/src/bootstrap/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs index 505f06ed1..e3fd942fc 100644 --- a/src/bootstrap/doc.rs +++ b/src/bootstrap/src/core/build_steps/doc.rs @@ -10,13 +10,13 @@ use std::fs; use std::path::{Path, PathBuf}; -use crate::builder::crate_description; -use crate::builder::{Alias, Builder, Compiler, Kind, RunConfig, ShouldRun, Step}; -use crate::cache::{Interned, INTERNER}; -use crate::compile; -use crate::config::{Config, TargetSelection}; -use crate::tool::{self, prepare_tool_cargo, SourceType, Tool}; -use crate::util::{dir_is_empty, symlink_dir, t, up_to_date}; +use crate::core::build_steps::compile; +use crate::core::build_steps::tool::{self, prepare_tool_cargo, SourceType, Tool}; +use crate::core::builder::crate_description; +use crate::core::builder::{Alias, Builder, Compiler, Kind, RunConfig, ShouldRun, Step}; +use crate::core::config::{Config, TargetSelection}; +use crate::utils::cache::{Interned, INTERNER}; +use crate::utils::helpers::{dir_is_empty, symlink_dir, t, up_to_date}; use crate::Mode; macro_rules! submodule_helper { @@ -685,19 +685,6 @@ impl Step for Rustc { target, ); - // This uses a shared directory so that librustdoc documentation gets - // correctly built and merged with the rustc documentation. This is - // needed because rustdoc is built in a different directory from - // rustc. rustdoc needs to be able to see everything, for example when - // merging the search index, or generating local (relative) links. - let out_dir = builder.stage_out(compiler, Mode::Rustc).join(target.triple).join("doc"); - t!(fs::create_dir_all(out_dir.parent().unwrap())); - symlink_dir_force(&builder.config, &out, &out_dir); - // Cargo puts proc macros in `target/doc` even if you pass `--target` - // explicitly (https://github.com/rust-lang/cargo/issues/7677). - let proc_macro_out_dir = builder.stage_out(compiler, Mode::Rustc).join("doc"); - symlink_dir_force(&builder.config, &out, &proc_macro_out_dir); - // Build cargo command. let mut cargo = builder.cargo(compiler, Mode::Rustc, SourceType::InTree, target, "doc"); cargo.rustdocflag("--document-private-items"); @@ -724,6 +711,7 @@ impl Step for Rustc { let mut to_open = None; + let out_dir = builder.stage_out(compiler, Mode::Rustc).join(target.triple).join("doc"); for krate in &*self.crates { // Create all crate output directories first to make sure rustdoc uses // relative links. @@ -736,8 +724,29 @@ impl Step for Rustc { } } + // This uses a shared directory so that librustdoc documentation gets + // correctly built and merged with the rustc documentation. + // + // This is needed because rustdoc is built in a different directory from + // rustc. rustdoc needs to be able to see everything, for example when + // merging the search index, or generating local (relative) links. + symlink_dir_force(&builder.config, &out, &out_dir); + // Cargo puts proc macros in `target/doc` even if you pass `--target` + // explicitly (https://github.com/rust-lang/cargo/issues/7677). + let proc_macro_out_dir = builder.stage_out(compiler, Mode::Rustc).join("doc"); + symlink_dir_force(&builder.config, &out, &proc_macro_out_dir); + builder.run(&mut cargo.into()); + if !builder.config.dry_run() { + // Sanity check on linked compiler crates + for krate in &*self.crates { + let dir_name = krate.replace("-", "_"); + // Making sure the directory exists and is not empty. + assert!(out.join(&*dir_name).read_dir().unwrap().next().is_some()); + } + } + if builder.paths.iter().any(|path| path.ends_with("compiler")) { // For `x.py doc compiler --open`, open `rustc_middle` by default. let index = out.join("rustc_middle").join("index.html"); @@ -756,10 +765,10 @@ macro_rules! tool_doc { $should_run: literal, $path: literal, $(rustc_tool = $rustc_tool:literal, )? - $(in_tree = $in_tree:literal, )? - [$($extra_arg: literal),+ $(,)?] - $(,)? - ) => { + $(in_tree = $in_tree:literal ,)? + $(is_library = $is_library:expr,)? + $(crates = $crates:expr)? + ) => { #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct $tool { target: TargetSelection, @@ -812,17 +821,6 @@ macro_rules! tool_doc { SourceType::Submodule }; - // Symlink compiler docs to the output directory of rustdoc documentation. - let out_dirs = [ - builder.stage_out(compiler, Mode::ToolRustc).join(target.triple).join("doc"), - // Cargo uses a different directory for proc macros. - builder.stage_out(compiler, Mode::ToolRustc).join("doc"), - ]; - for out_dir in out_dirs { - t!(fs::create_dir_all(&out_dir)); - symlink_dir_force(&builder.config, &out, &out_dir); - } - // Build cargo command. let mut cargo = prepare_tool_cargo( builder, @@ -839,9 +837,13 @@ macro_rules! tool_doc { // Only include compiler crates, no dependencies of those, such as `libc`. cargo.arg("--no-deps"); - $( - cargo.arg($extra_arg); - )+ + if false $(|| $is_library)? { + cargo.arg("--lib"); + } + + $(for krate in $crates { + cargo.arg("-p").arg(krate); + })? cargo.rustdocflag("--document-private-items"); // Since we always pass --document-private-items, there's no need to warn about linking to private items. @@ -851,62 +853,69 @@ macro_rules! tool_doc { cargo.rustdocflag("--generate-link-to-definition"); cargo.rustdocflag("-Zunstable-options"); + let out_dir = builder.stage_out(compiler, Mode::ToolRustc).join(target.triple).join("doc"); + $(for krate in $crates { + let dir_name = krate.replace("-", "_"); + t!(fs::create_dir_all(out_dir.join(&*dir_name))); + })? + + // Symlink compiler docs to the output directory of rustdoc documentation. + symlink_dir_force(&builder.config, &out, &out_dir); + let proc_macro_out_dir = builder.stage_out(compiler, Mode::ToolRustc).join("doc"); + symlink_dir_force(&builder.config, &out, &proc_macro_out_dir); + let _guard = builder.msg_doc(compiler, stringify!($tool).to_lowercase(), target); builder.run(&mut cargo.into()); + + if !builder.config.dry_run() { + // Sanity check on linked doc directories + $(for krate in $crates { + let dir_name = krate.replace("-", "_"); + // Making sure the directory exists and is not empty. + assert!(out.join(&*dir_name).read_dir().unwrap().next().is_some()); + })? + } } } } } -tool_doc!( - Rustdoc, - "rustdoc-tool", - "src/tools/rustdoc", - ["-p", "rustdoc", "-p", "rustdoc-json-types"] -); +tool_doc!(Rustdoc, "rustdoc-tool", "src/tools/rustdoc", crates = ["rustdoc", "rustdoc-json-types"]); tool_doc!( Rustfmt, "rustfmt-nightly", "src/tools/rustfmt", - ["-p", "rustfmt-nightly", "-p", "rustfmt-config_proc_macro"], + crates = ["rustfmt-nightly", "rustfmt-config_proc_macro"] ); -tool_doc!(Clippy, "clippy", "src/tools/clippy", ["-p", "clippy_utils"]); -tool_doc!(Miri, "miri", "src/tools/miri", ["-p", "miri"]); +tool_doc!(Clippy, "clippy", "src/tools/clippy", crates = ["clippy_config", "clippy_utils"]); +tool_doc!(Miri, "miri", "src/tools/miri", crates = ["miri"]); tool_doc!( Cargo, "cargo", "src/tools/cargo", rustc_tool = false, in_tree = false, - [ - "-p", + crates = [ "cargo", - "-p", "cargo-platform", - "-p", "cargo-util", - "-p", "crates-io", - "-p", "cargo-test-macro", - "-p", "cargo-test-support", - "-p", "cargo-credential", - "-p", "mdman", // FIXME: this trips a license check in tidy. - // "-p", // "resolver-tests", ] ); -tool_doc!(Tidy, "tidy", "src/tools/tidy", rustc_tool = false, ["-p", "tidy"]); +tool_doc!(Tidy, "tidy", "src/tools/tidy", rustc_tool = false, crates = ["tidy"]); tool_doc!( Bootstrap, "bootstrap", "src/bootstrap", rustc_tool = false, - ["--lib", "-p", "bootstrap"] + is_library = true, + crates = ["bootstrap"] ); #[derive(Ord, PartialOrd, Debug, Copy, Clone, Hash, PartialEq, Eq)] diff --git a/src/bootstrap/format.rs b/src/bootstrap/src/core/build_steps/format.rs index 11f2762f7..86f1d925f 100644 --- a/src/bootstrap/format.rs +++ b/src/bootstrap/src/core/build_steps/format.rs @@ -1,7 +1,7 @@ //! Runs rustfmt on the repository. -use crate::builder::Builder; -use crate::util::{output, program_out_of_date, t}; +use crate::core::builder::Builder; +use crate::utils::helpers::{output, program_out_of_date, t}; use build_helper::ci::CiEnv; use build_helper::git::get_git_modified_files; use ignore::WalkBuilder; @@ -89,7 +89,7 @@ fn get_modified_rs_files(build: &Builder<'_>) -> Result<Option<Vec<String>>, Str return Ok(None); } - get_git_modified_files(Some(&build.config.src), &vec!["rs"]) + get_git_modified_files(&build.config.git_config(), Some(&build.config.src), &vec!["rs"]) } #[derive(serde_derive::Deserialize)] diff --git a/src/bootstrap/install.rs b/src/bootstrap/src/core/build_steps/install.rs index 500b20b86..6b4a8f597 100644 --- a/src/bootstrap/install.rs +++ b/src/bootstrap/src/core/build_steps/install.rs @@ -8,15 +8,14 @@ use std::fs; use std::path::{Component, Path, PathBuf}; use std::process::Command; -use crate::util::t; - -use crate::dist; -use crate::tarball::GeneratedTarball; +use crate::core::build_steps::dist; +use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::core::config::{Config, TargetSelection}; +use crate::utils::helpers::t; +use crate::utils::tarball::GeneratedTarball; +use crate::INTERNER; use crate::{Compiler, Kind}; -use crate::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::config::{Config, TargetSelection}; - #[cfg(target_os = "illumos")] const SHELL: &str = "bash"; #[cfg(not(target_os = "illumos"))] @@ -270,7 +269,7 @@ install!((self, builder, _config), } }; RustDemangler, alias = "rust-demangler", Self::should_build(_config), only_hosts: true, { - // Note: Even though `should_build` may return true for `extended` default tools, + // NOTE: Even though `should_build` may return true for `extended` default tools, // dist::RustDemangler may still return None, unless the target-dependent `profiler` config // is also true, or the `tools` array explicitly includes "rust-demangler". if let Some(tarball) = builder.ensure(dist::RustDemangler { @@ -291,6 +290,19 @@ install!((self, builder, _config), }); install_sh(builder, "rustc", self.compiler.stage, Some(self.target), &tarball); }; + RustcCodegenCranelift, alias = "rustc-codegen-cranelift", Self::should_build(_config), only_hosts: true, { + if let Some(tarball) = builder.ensure(dist::CodegenBackend { + compiler: self.compiler, + backend: INTERNER.intern_str("cranelift"), + }) { + install_sh(builder, "rustc-codegen-cranelift", self.compiler.stage, Some(self.target), &tarball); + } else { + builder.info( + &format!("skipping Install CodegenBackend(\"cranelift\") stage{} ({})", + self.compiler.stage, self.target), + ); + } + }; ); #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] diff --git a/src/bootstrap/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 455683158..a1f6fac8a 100644 --- a/src/bootstrap/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -16,11 +16,10 @@ use std::io; use std::path::{Path, PathBuf}; use std::process::Command; -use crate::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::channel; -use crate::config::{Config, TargetSelection}; -use crate::util::get_clang_cl_resource_dir; -use crate::util::{self, exe, output, t, up_to_date}; +use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::core::config::{Config, TargetSelection}; +use crate::utils::channel; +use crate::utils::helpers::{self, exe, get_clang_cl_resource_dir, output, t, up_to_date}; use crate::{CLang, GitRepo, Kind}; use build_helper::ci::CiEnv; @@ -133,8 +132,8 @@ pub(crate) fn detect_llvm_sha(config: &Config, is_git: bool) -> String { // walk back further to the last bors merge commit that actually changed LLVM. The first // step will fail on CI because only the `auto` branch exists; we just fall back to `HEAD` // in that case. - let closest_upstream = - get_git_merge_base(Some(&config.src)).unwrap_or_else(|_| "HEAD".into()); + let closest_upstream = get_git_merge_base(&config.git_config(), Some(&config.src)) + .unwrap_or_else(|_| "HEAD".into()); let mut rev_list = config.git(); rev_list.args(&[ PathBuf::from("rev-list"), @@ -157,9 +156,9 @@ pub(crate) fn detect_llvm_sha(config: &Config, is_git: bool) -> String { if llvm_sha.is_empty() { eprintln!("error: could not find commit hash for downloading LLVM"); - eprintln!("help: maybe your repository history is too shallow?"); - eprintln!("help: consider disabling `download-ci-llvm`"); - eprintln!("help: or fetch enough history to include one upstream commit"); + eprintln!("HELP: maybe your repository history is too shallow?"); + eprintln!("HELP: consider disabling `download-ci-llvm`"); + eprintln!("HELP: or fetch enough history to include one upstream commit"); panic!(); } @@ -281,7 +280,7 @@ impl Step for Llvm { let _guard = builder.msg_unstaged(Kind::Build, "LLVM", target); t!(stamp.remove()); - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); t!(fs::create_dir_all(&out_dir)); // https://llvm.org/docs/CMake.html @@ -397,6 +396,12 @@ impl Step for Llvm { ldflags.shared.push(" -latomic"); } + if target.starts_with("mips") && target.contains("netbsd") { + // LLVM wants 64-bit atomics, while mipsel is 32-bit only, so needs -latomic + ldflags.exe.push(" -latomic"); + ldflags.shared.push(" -latomic"); + } + if target.contains("msvc") { cfg.define("LLVM_USE_CRT_DEBUG", "MT"); cfg.define("LLVM_USE_CRT_RELEASE", "MT"); @@ -410,7 +415,7 @@ impl Step for Llvm { let mut enabled_llvm_projects = Vec::new(); - if util::forcing_clang_based_tests() { + if helpers::forcing_clang_based_tests() { enabled_llvm_projects.push("clang"); enabled_llvm_projects.push("compiler-rt"); } @@ -528,8 +533,12 @@ impl Step for Llvm { // If the shared library exists in LLVM's `/build/lib/` or `/lib/` folders, strip its // debuginfo. - crate::compile::strip_debug(builder, target, &out_dir.join("lib").join(&lib_name)); - crate::compile::strip_debug( + crate::core::build_steps::compile::strip_debug( + builder, + target, + &out_dir.join("lib").join(&lib_name), + ); + crate::core::build_steps::compile::strip_debug( builder, target, &out_dir.join("build").join("lib").join(&lib_name), @@ -846,7 +855,7 @@ impl Step for Lld { } let _guard = builder.msg_unstaged(Kind::Build, "LLD", target); - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); t!(fs::create_dir_all(&out_dir)); let mut cfg = cmake::Config::new(builder.src.join("src/llvm-project/lld")); @@ -877,7 +886,7 @@ impl Step for Lld { // `LD_LIBRARY_PATH` overrides) // if builder.config.rpath_enabled(target) - && util::use_host_linker(target) + && helpers::use_host_linker(target) && builder.config.llvm_link_shared() && target.contains("linux") { @@ -970,7 +979,7 @@ impl Step for Sanitizers { let _guard = builder.msg_unstaged(Kind::Build, "sanitizers", self.target); t!(stamp.remove()); - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); let mut cfg = cmake::Config::new(&compiler_rt_dir); cfg.profile("Release"); diff --git a/src/bootstrap/src/core/build_steps/mod.rs b/src/bootstrap/src/core/build_steps/mod.rs new file mode 100644 index 000000000..50d83789b --- /dev/null +++ b/src/bootstrap/src/core/build_steps/mod.rs @@ -0,0 +1,15 @@ +pub(crate) mod check; +pub(crate) mod clean; +pub(crate) mod compile; +pub(crate) mod dist; +pub(crate) mod doc; +pub(crate) mod format; +pub(crate) mod install; +pub(crate) mod llvm; +pub(crate) mod run; +pub(crate) mod setup; +pub(crate) mod suggest; +pub(crate) mod synthetic_targets; +pub(crate) mod test; +pub(crate) mod tool; +pub(crate) mod toolstate; diff --git a/src/bootstrap/run.rs b/src/bootstrap/src/core/build_steps/run.rs index 4082f5bb9..d1d6b7e86 100644 --- a/src/bootstrap/run.rs +++ b/src/bootstrap/src/core/build_steps/run.rs @@ -1,15 +1,13 @@ use std::path::PathBuf; use std::process::Command; -use clap_complete::shells; - -use crate::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::config::TargetSelection; -use crate::dist::distdir; -use crate::flags::get_completion; -use crate::test; -use crate::tool::{self, SourceType, Tool}; -use crate::util::output; +use crate::core::build_steps::dist::distdir; +use crate::core::build_steps::test; +use crate::core::build_steps::tool::{self, SourceType, Tool}; +use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::core::config::flags::get_completion; +use crate::core::config::TargetSelection; +use crate::utils::helpers::output; use crate::Mode; #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] @@ -268,23 +266,29 @@ impl Step for GenerateWindowsSys { #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] pub struct GenerateCompletions; +macro_rules! generate_completions { + ( $( ( $shell:ident, $filename:expr ) ),* ) => { + $( + if let Some(comp) = get_completion($shell, &$filename) { + std::fs::write(&$filename, comp).expect(&format!("writing {} completion", stringify!($shell))); + } + )* + }; +} + impl Step for GenerateCompletions { type Output = (); /// Uses `clap_complete` to generate shell completions. fn run(self, builder: &Builder<'_>) { - // FIXME(clubby789): enable zsh when clap#4898 is fixed - let [bash, fish, powershell] = ["x.py.sh", "x.py.fish", "x.py.ps1"] - .map(|filename| builder.src.join("src/etc/completions").join(filename)); - if let Some(comp) = get_completion(shells::Bash, &bash) { - std::fs::write(&bash, comp).expect("writing bash completion"); - } - if let Some(comp) = get_completion(shells::Fish, &fish) { - std::fs::write(&fish, comp).expect("writing fish completion"); - } - if let Some(comp) = get_completion(shells::PowerShell, &powershell) { - std::fs::write(&powershell, comp).expect("writing powershell completion"); - } + use clap_complete::shells::{Bash, Fish, PowerShell, Zsh}; + + generate_completions!( + (Bash, builder.src.join("src/etc/completions/x.py.sh")), + (Zsh, builder.src.join("src/etc/completions/x.py.zsh")), + (Fish, builder.src.join("src/etc/completions/x.py.fish")), + (PowerShell, builder.src.join("src/etc/completions/x.py.ps1")) + ); } fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { diff --git a/src/bootstrap/setup.rs b/src/bootstrap/src/core/build_steps/setup.rs index ef0234957..486a1e20f 100644 --- a/src/bootstrap/setup.rs +++ b/src/bootstrap/src/core/build_steps/setup.rs @@ -1,6 +1,6 @@ -use crate::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::Config; -use crate::{t, VERSION}; +use crate::{t, CONFIG_CHANGE_HISTORY}; use sha2::Digest; use std::env::consts::EXE_SUFFIX; use std::fmt::Write as _; @@ -12,6 +12,7 @@ use std::str::FromStr; use std::{fmt, fs, io}; #[cfg(test)] +#[path = "../../tests/setup.rs"] mod tests; #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] @@ -35,7 +36,7 @@ static SETTINGS_HASHES: &[&str] = &[ "47d227f424bf889b0d899b9cc992d5695e1b78c406e183cd78eafefbe5488923", "b526bd58d0262dd4dda2bff5bc5515b705fb668a46235ace3e057f807963a11a", ]; -static RUST_ANALYZER_SETTINGS: &str = include_str!("../etc/rust_analyzer_settings.json"); +static RUST_ANALYZER_SETTINGS: &str = include_str!("../../../../etc/rust_analyzer_settings.json"); impl Profile { fn include_path(&self, src_path: &Path) -> PathBuf { @@ -121,6 +122,27 @@ impl Step for Profile { return; } + let path = &run.builder.config.config.clone().unwrap_or(PathBuf::from("config.toml")); + if path.exists() { + eprintln!(); + eprintln!( + "ERROR: you asked for a new config file, but one already exists at `{}`", + t!(path.canonicalize()).display() + ); + + match prompt_user( + "Do you wish to override the existing configuration (which will allow the setup process to continue)?: [y/N]", + ) { + Ok(Some(PromptResult::Yes)) => { + t!(fs::remove_file(path)); + } + _ => { + println!("Exiting."); + crate::exit!(1); + } + } + } + // for Profile, `run.paths` will have 1 and only 1 element // this is because we only accept at most 1 path from user input. // If user calls `x.py setup` without arguments, the interactive TUI @@ -146,6 +168,15 @@ impl Step for Profile { } fn run(self, builder: &Builder<'_>) { + // During ./x.py setup once you select the codegen profile. + // The submodule will be downloaded. It does not work in the + // tarball case since they don't include Git and submodules + // are already included. + if !builder.rust_info().is_from_tarball() { + if self == Profile::Codegen { + builder.update_submodule(&Path::new("src/llvm-project")); + } + } setup(&builder.build.config, self) } } @@ -181,8 +212,8 @@ pub fn setup(config: &Config, profile: Profile) { if profile == Profile::Tools { eprintln!(); eprintln!( - "note: the `tools` profile sets up the `stage2` toolchain (use \ - `rustup toolchain link 'name' host/build/stage2` to use rustc)" + "NOTE: the `tools` profile sets up the `stage2` toolchain (use \ + `rustup toolchain link 'name' build/host/stage2` to use rustc)" ) } @@ -194,24 +225,12 @@ fn setup_config_toml(path: &PathBuf, profile: Profile, config: &Config) { if profile == Profile::None { return; } - if path.exists() { - eprintln!(); - eprintln!( - "error: you asked `x.py` to setup a new config file, but one already exists at `{}`", - path.display() - ); - eprintln!("help: try adding `profile = \"{}\"` at the top of {}", profile, path.display()); - eprintln!( - "note: this will use the configuration in {}", - profile.include_path(&config.src).display() - ); - crate::exit!(1); - } + let latest_change_id = CONFIG_CHANGE_HISTORY.last().unwrap(); let settings = format!( "# Includes one of the default files in src/bootstrap/defaults\n\ profile = \"{profile}\"\n\ - changelog-seen = {VERSION}\n" + change-id = {latest_change_id}\n" ); t!(fs::write(path, settings)); @@ -395,8 +414,8 @@ pub fn interactive_path() -> io::Result<Profile> { break match parse_with_abbrev(&input) { Ok(profile) => profile, Err(err) => { - eprintln!("error: {err}"); - eprintln!("note: press Ctrl+C to exit"); + eprintln!("ERROR: {err}"); + eprintln!("NOTE: press Ctrl+C to exit"); continue; } }; @@ -425,8 +444,8 @@ fn prompt_user(prompt: &str) -> io::Result<Option<PromptResult>> { "p" | "print" => return Ok(Some(PromptResult::Print)), "" => return Ok(None), _ => { - eprintln!("error: unrecognized option '{}'", input.trim()); - eprintln!("note: press Ctrl+C to exit"); + eprintln!("ERROR: unrecognized option '{}'", input.trim()); + eprintln!("NOTE: press Ctrl+C to exit"); } }; } @@ -467,7 +486,8 @@ fn install_git_hook_maybe(config: &Config) -> io::Result<()> { assert!(output.status.success(), "failed to run `git`"); PathBuf::from(t!(String::from_utf8(output.stdout)).trim()) })); - let dst = git.join("hooks").join("pre-push"); + let hooks_dir = git.join("hooks"); + let dst = hooks_dir.join("pre-push"); if dst.exists() { // The git hook has already been set up, or the user already has a custom hook. return Ok(()); @@ -484,11 +504,15 @@ undesirable, simply delete the `pre-push` file from .git/hooks." println!("Ok, skipping installation!"); return Ok(()); } + if !hooks_dir.exists() { + // We need to (try to) create the hooks directory first. + let _ = fs::create_dir(hooks_dir); + } let src = config.src.join("src").join("etc").join("pre-push.sh"); match fs::hard_link(src, &dst) { Err(e) => { eprintln!( - "error: could not create hook {}: do you already have the git hook installed?\n{}", + "ERROR: could not create hook {}: do you already have the git hook installed?\n{}", dst.display(), e ); @@ -554,10 +578,10 @@ fn create_vscode_settings_maybe(config: &Config) -> io::Result<()> { ); match mismatched_settings { Some(true) => eprintln!( - "warning: existing `.vscode/settings.json` is out of date, x.py will update it" + "WARNING: existing `.vscode/settings.json` is out of date, x.py will update it" ), Some(false) => eprintln!( - "warning: existing `.vscode/settings.json` has been modified by user, x.py will back it up and replace it" + "WARNING: existing `.vscode/settings.json` has been modified by user, x.py will back it up and replace it" ), _ => (), } @@ -584,7 +608,7 @@ fn create_vscode_settings_maybe(config: &Config) -> io::Result<()> { // exists and is not current version or outdated, so back it up let mut backup = vscode_settings.clone(); backup.set_extension("json.bak"); - eprintln!("warning: copying `settings.json` to `settings.json.bak`"); + eprintln!("WARNING: copying `settings.json` to `settings.json.bak`"); fs::copy(&vscode_settings, &backup)?; "Updated" } diff --git a/src/bootstrap/suggest.rs b/src/bootstrap/src/core/build_steps/suggest.rs index f225104bd..93da27560 100644 --- a/src/bootstrap/suggest.rs +++ b/src/bootstrap/src/core/build_steps/suggest.rs @@ -1,17 +1,21 @@ #![cfg_attr(feature = "build-metrics", allow(unused))] -use std::str::FromStr; - -use std::path::PathBuf; - use clap::Parser; +use std::path::PathBuf; +use std::str::FromStr; -use crate::{builder::Builder, tool::Tool}; +use crate::core::build_steps::tool::Tool; +use crate::core::builder::Builder; /// Suggests a list of possible `x.py` commands to run based on modified files in branch. pub fn suggest(builder: &Builder<'_>, run: bool) { - let suggestions = - builder.tool_cmd(Tool::SuggestTests).output().expect("failed to run `suggest-tests` tool"); + let git_config = builder.config.git_config(); + let suggestions = builder + .tool_cmd(Tool::SuggestTests) + .env("SUGGEST_TESTS_GIT_REPOSITORY", git_config.git_repository) + .env("SUGGEST_TESTS_NIGHTLY_BRANCH", git_config.nightly_branch) + .output() + .expect("failed to run `suggest-tests` tool"); if !suggestions.status.success() { println!("failed to run `suggest-tests` tool ({})", suggestions.status); @@ -62,13 +66,13 @@ pub fn suggest(builder: &Builder<'_>, run: bool) { for sug in suggestions { let mut build: crate::Build = builder.build.clone(); build.config.paths = sug.2; - build.config.cmd = crate::flags::Flags::parse_from(["x.py", sug.0]).cmd; + build.config.cmd = crate::core::config::flags::Flags::parse_from(["x.py", sug.0]).cmd; if let Some(stage) = sug.1 { build.config.stage = stage; } build.build(); } } else { - println!("help: consider using the `--run` flag to automatically run suggested tests"); + println!("HELP: consider using the `--run` flag to automatically run suggested tests"); } } diff --git a/src/bootstrap/synthetic_targets.rs b/src/bootstrap/src/core/build_steps/synthetic_targets.rs index 7eeac9025..d2c65b740 100644 --- a/src/bootstrap/synthetic_targets.rs +++ b/src/bootstrap/src/core/build_steps/synthetic_targets.rs @@ -7,8 +7,8 @@ //! one of the target specs already defined in this module, or create new ones by adding a new step //! that calls create_synthetic_target. -use crate::builder::{Builder, ShouldRun, Step}; -use crate::config::TargetSelection; +use crate::core::builder::{Builder, ShouldRun, Step}; +use crate::core::config::TargetSelection; use crate::Compiler; use std::process::{Command, Stdio}; @@ -76,7 +76,7 @@ fn create_synthetic_target( std::fs::write(&path, &serde_json::to_vec_pretty(&spec).unwrap()).unwrap(); let target = TargetSelection::create_synthetic(&name, path.to_str().unwrap()); - crate::cc_detect::find_target(builder, target); + crate::utils::cc_detect::find_target(builder, target); target } diff --git a/src/bootstrap/test.rs b/src/bootstrap/src/core/build_steps/test.rs index ba030f0f5..d2aa89dee 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -13,21 +13,26 @@ use std::process::{Command, Stdio}; use clap_complete::shells; -use crate::builder::crate_description; -use crate::builder::{Builder, Compiler, Kind, RunConfig, ShouldRun, Step}; -use crate::cache::Interned; -use crate::cache::INTERNER; -use crate::compile; -use crate::config::TargetSelection; -use crate::dist; -use crate::doc::DocumentationFormat; -use crate::flags::Subcommand; -use crate::llvm; -use crate::render_tests::add_flags_and_try_run_tests; -use crate::synthetic_targets::MirOptPanicAbortSyntheticTarget; -use crate::tool::{self, SourceType, Tool}; -use crate::toolstate::ToolState; -use crate::util::{self, add_link_lib_path, dylib_path, dylib_path_var, output, t, up_to_date}; +use crate::core::build_steps::compile; +use crate::core::build_steps::dist; +use crate::core::build_steps::doc::DocumentationFormat; +use crate::core::build_steps::llvm; +use crate::core::build_steps::synthetic_targets::MirOptPanicAbortSyntheticTarget; +use crate::core::build_steps::tool::{self, SourceType, Tool}; +use crate::core::build_steps::toolstate::ToolState; +use crate::core::builder::crate_description; +use crate::core::builder::{Builder, Compiler, Kind, RunConfig, ShouldRun, Step}; +use crate::core::config::flags::get_completion; +use crate::core::config::flags::Subcommand; +use crate::core::config::TargetSelection; +use crate::utils; +use crate::utils::cache::{Interned, INTERNER}; +use crate::utils::exec::BootstrapCommand; +use crate::utils::helpers::{ + self, add_link_lib_path, dylib_path, dylib_path_var, output, t, + target_supports_cranelift_backend, up_to_date, +}; +use crate::utils::render_tests::{add_flags_and_try_run_tests, try_run_tests}; use crate::{envify, CLang, DocTests, GitRepo, Mode}; const ADB_TEST_DIR: &str = "/data/local/tmp/work"; @@ -167,7 +172,7 @@ You can skip linkcheck with --skip src/tools/linkchecker" // Run the linkchecker. let _guard = builder.msg(Kind::Test, compiler.stage, "Linkcheck", bootstrap_host, bootstrap_host); - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); builder.run_delaying_failure(linkchecker.arg(builder.out.join(host.triple).join("doc"))); } @@ -219,7 +224,11 @@ impl Step for HtmlCheck { } // Ensure that a few different kinds of documentation are available. builder.default_doc(&[]); - builder.ensure(crate::doc::Rustc::new(builder.top_stage, self.target, builder)); + builder.ensure(crate::core::build_steps::doc::Rustc::new( + builder.top_stage, + self.target, + builder, + )); builder.run_delaying_failure( builder.tool_cmd(Tool::HtmlChecker).arg(builder.doc_out(self.target)), @@ -260,7 +269,7 @@ impl Step for Cargotest { let out_dir = builder.out.join("ct"); t!(fs::create_dir_all(&out_dir)); - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); let mut cmd = builder.tool_cmd(Tool::CargoTest); builder.run_delaying_failure( cmd.arg(&cargo) @@ -328,7 +337,7 @@ impl Step for Cargo { builder, ); - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); add_flags_and_try_run_tests(builder, &mut cargo); } } @@ -622,7 +631,7 @@ impl Step for Miri { SourceType::InTree, &[], ); - let _guard = builder.msg_sysroot_tool(Kind::Test, compiler.stage, "miri", host, host); + let _guard = builder.msg_sysroot_tool(Kind::Test, compiler.stage, "miri", host, target); cargo.add_rustc_lib_path(builder, compiler); @@ -642,7 +651,7 @@ impl Step for Miri { // does not understand the flags added by `add_flags_and_try_run_test`. let mut cargo = prepare_cargo_test(cargo, &[], &[], "miri", compiler, target, builder); { - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); builder.run(&mut cargo); } @@ -658,7 +667,7 @@ impl Step for Miri { let mut cargo = prepare_cargo_test(cargo, &[], &[], "miri", compiler, target, builder); { - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); builder.run(&mut cargo); } } @@ -698,7 +707,7 @@ impl Step for Miri { let mut cargo = Command::from(cargo); { - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); builder.run(&mut cargo); } } @@ -801,8 +810,8 @@ impl Step for Clippy { let _guard = builder.msg_sysroot_tool(Kind::Test, compiler.stage, "clippy", host, host); - #[allow(deprecated)] // Clippy reports errors if it blessed the outputs - if builder.config.try_run(&mut cargo).is_ok() { + // Clippy reports errors if it blessed the outputs + if builder.run_cmd(BootstrapCommand::from(&mut cargo).allow_failure()) { // The tests succeeded; nothing to do. return; } @@ -859,7 +868,7 @@ impl Step for RustdocTheme { if builder.is_fuse_ld_lld(self.compiler.host) { cmd.env( "RUSTDOC_LLD_NO_THREADS", - util::lld_flag_no_threads(self.compiler.host.contains("windows")), + helpers::lld_flag_no_threads(self.compiler.host.contains("windows")), ); } builder.run_delaying_failure(&mut cmd); @@ -900,7 +909,8 @@ impl Step for RustdocJSStd { .arg("--test-folder") .arg(builder.src.join("tests/rustdoc-js-std")); for path in &builder.paths { - if let Some(p) = util::is_valid_test_suite_arg(path, "tests/rustdoc-js-std", builder) { + if let Some(p) = helpers::is_valid_test_suite_arg(path, "tests/rustdoc-js-std", builder) + { if !p.ends_with(".js") { eprintln!("A non-js file was given: `{}`", path.display()); panic!("Cannot run rustdoc-js-std tests"); @@ -908,7 +918,7 @@ impl Step for RustdocJSStd { command.arg("--test-file").arg(path); } } - builder.ensure(crate::doc::Std::new( + builder.ensure(crate::core::build_steps::doc::Std::new( builder.top_stage, self.target, builder, @@ -1035,7 +1045,7 @@ impl Step for RustdocGUI { .env("RUSTC", builder.rustc(self.compiler)); for path in &builder.paths { - if let Some(p) = util::is_valid_test_suite_arg(path, "tests/rustdoc-gui", builder) { + if let Some(p) = helpers::is_valid_test_suite_arg(path, "tests/rustdoc-gui", builder) { if !p.ends_with(".goml") { eprintln!("A non-goml file was given: `{}`", path.display()); panic!("Cannot run rustdoc-gui tests"); @@ -1058,7 +1068,7 @@ impl Step for RustdocGUI { cmd.arg("--npm").arg(npm); } - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); let _guard = builder.msg_sysroot_tool( Kind::Test, self.compiler.stage, @@ -1066,7 +1076,7 @@ impl Step for RustdocGUI { self.compiler.host, self.target, ); - crate::render_tests::try_run_tests(builder, &mut cmd, true); + try_run_tests(builder, &mut cmd, true); } } @@ -1117,16 +1127,16 @@ impl Step for Tidy { let inferred_rustfmt_dir = builder.initial_rustc.parent().unwrap(); eprintln!( "\ -error: no `rustfmt` binary found in {PATH} -info: `rust.channel` is currently set to \"{CHAN}\" -help: if you are testing a beta branch, set `rust.channel` to \"beta\" in the `config.toml` file -help: to skip test's attempt to check tidiness, pass `--skip src/tools/tidy` to `x.py test`", +ERROR: no `rustfmt` binary found in {PATH} +INFO: `rust.channel` is currently set to \"{CHAN}\" +HELP: if you are testing a beta branch, set `rust.channel` to \"beta\" in the `config.toml` file +HELP: to skip test's attempt to check tidiness, pass `--skip src/tools/tidy` to `x.py test`", PATH = inferred_rustfmt_dir.display(), CHAN = builder.config.channel, ); crate::exit!(1); } - crate::format::format(&builder, !builder.config.cmd.bless(), &[]); + crate::core::build_steps::format::format(&builder, !builder.config.cmd.bless(), &[]); } builder.info("tidy check"); @@ -1135,13 +1145,14 @@ help: to skip test's attempt to check tidiness, pass `--skip src/tools/tidy` to builder.ensure(ExpandYamlAnchors); builder.info("x.py completions check"); - let [bash, fish, powershell] = ["x.py.sh", "x.py.fish", "x.py.ps1"] + let [bash, zsh, fish, powershell] = ["x.py.sh", "x.py.zsh", "x.py.fish", "x.py.ps1"] .map(|filename| builder.src.join("src/etc/completions").join(filename)); if builder.config.cmd.bless() { - builder.ensure(crate::run::GenerateCompletions); - } else if crate::flags::get_completion(shells::Bash, &bash).is_some() - || crate::flags::get_completion(shells::Fish, &fish).is_some() - || crate::flags::get_completion(shells::PowerShell, &powershell).is_some() + builder.ensure(crate::core::build_steps::run::GenerateCompletions); + } else if get_completion(shells::Bash, &bash).is_some() + || get_completion(shells::Fish, &fish).is_some() + || get_completion(shells::PowerShell, &powershell).is_some() + || crate::flags::get_completion(shells::Zsh, &zsh).is_some() { eprintln!( "x.py completions were changed; run `x.py run generate-completions` to update them" @@ -1172,7 +1183,7 @@ impl Step for ExpandYamlAnchors { /// appropriate configuration for all our CI providers. This step ensures the tool was called /// by the user before committing CI changes. fn run(self, builder: &Builder<'_>) { - // Note: `.github/` is not included in dist-src tarballs + // NOTE: `.github/` is not included in dist-src tarballs if !builder.src.join(".github/workflows/ci.yml").exists() { builder.info("Skipping YAML anchors check: GitHub Actions config not found"); return; @@ -1294,6 +1305,47 @@ macro_rules! test_definitions { }; } +/// Declares an alias for running the [`Coverage`] tests in only one mode. +/// Adapted from [`test_definitions`]. +macro_rules! coverage_test_alias { + ($name:ident { + alias_and_mode: $alias_and_mode:expr, + default: $default:expr, + only_hosts: $only_hosts:expr $(,)? + }) => { + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] + pub struct $name { + pub compiler: Compiler, + pub target: TargetSelection, + } + + impl $name { + const MODE: &'static str = $alias_and_mode; + } + + impl Step for $name { + type Output = (); + const DEFAULT: bool = $default; + const ONLY_HOSTS: bool = $only_hosts; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias($alias_and_mode) + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + + run.builder.ensure($name { compiler, target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + Coverage { compiler: self.compiler, target: self.target } + .run_unified_suite(builder, Self::MODE) + } + } + }; +} + default_test!(Ui { path: "tests/ui", mode: "ui", suite: "ui" }); default_test!(RunPassValgrind { @@ -1338,17 +1390,70 @@ host_test!(RunMakeFullDeps { default_test!(Assembly { path: "tests/assembly", mode: "assembly", suite: "assembly" }); -default_test!(CoverageMap { - path: "tests/coverage-map", - mode: "coverage-map", - suite: "coverage-map" +/// Custom test step that is responsible for running the coverage tests +/// in multiple different modes. +/// +/// Each individual mode also has its own alias that will run the tests in +/// just that mode. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Coverage { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Coverage { + const PATH: &'static str = "tests/coverage"; + const SUITE: &'static str = "coverage"; + + fn run_unified_suite(&self, builder: &Builder<'_>, mode: &'static str) { + builder.ensure(Compiletest { + compiler: self.compiler, + target: self.target, + mode, + suite: Self::SUITE, + path: Self::PATH, + compare_mode: None, + }) + } +} + +impl Step for Coverage { + type Output = (); + const DEFAULT: bool = false; + const ONLY_HOSTS: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.suite_path(Self::PATH) + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + + run.builder.ensure(Coverage { compiler, target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + self.run_unified_suite(builder, CoverageMap::MODE); + self.run_unified_suite(builder, CoverageRun::MODE); + } +} + +// Aliases for running the coverage tests in only one mode. +coverage_test_alias!(CoverageMap { + alias_and_mode: "coverage-map", + default: true, + only_hosts: false, +}); +coverage_test_alias!(CoverageRun { + alias_and_mode: "coverage-run", + default: true, + only_hosts: true, }); -host_test!(RunCoverage { path: "tests/run-coverage", mode: "run-coverage", suite: "run-coverage" }); -host_test!(RunCoverageRustdoc { - path: "tests/run-coverage-rustdoc", - mode: "run-coverage", - suite: "run-coverage-rustdoc" +host_test!(CoverageRunRustdoc { + path: "tests/coverage-run-rustdoc", + mode: "coverage-run", + suite: "coverage-run-rustdoc" }); // For the mir-opt suite we do not use macros, as we need custom behavior when blessing. @@ -1402,10 +1507,10 @@ impl Step for MirOpt { // have been detected by bootstrap if the target we're testing wasn't in the // --target flags. if !builder.cc.borrow().contains_key(&target_32bit) { - crate::cc_detect::find_target(builder, target_32bit); + utils::cc_detect::find_target(builder, target_32bit); } if !builder.cc.borrow().contains_key(&target_64bit) { - crate::cc_detect::find_target(builder, target_64bit); + utils::cc_detect::find_target(builder, target_64bit); } vec![target_32bit, target_64bit] @@ -1461,10 +1566,10 @@ impl Step for Compiletest { fn run(self, builder: &Builder<'_>) { if builder.top_stage == 0 && env::var("COMPILETEST_FORCE_STAGE0").is_err() { eprintln!("\ -error: `--stage 0` runs compiletest on the beta compiler, not your local changes, and will almost always cause tests to fail -help: to test the compiler, use `--stage 1` instead -help: to test the standard library, use `--stage 0 library/std` instead -note: if you're sure you want to do this, please open an issue as to why. In the meantime, you can override this with `COMPILETEST_FORCE_STAGE0=1`." +ERROR: `--stage 0` runs compiletest on the beta compiler, not your local changes, and will almost always cause tests to fail +HELP: to test the compiler, use `--stage 1` instead +HELP: to test the standard library, use `--stage 0 library/std` instead +NOTE: if you're sure you want to do this, please open an issue as to why. In the meantime, you can override this with `COMPILETEST_FORCE_STAGE0=1`." ); crate::exit!(1); } @@ -1535,7 +1640,7 @@ note: if you're sure you want to do this, please open an issue as to why. In the || (mode == "ui" && is_rustdoc) || mode == "js-doc-test" || mode == "rustdoc-json" - || suite == "run-coverage-rustdoc" + || suite == "coverage-run-rustdoc" { cmd.arg("--rustdoc-path").arg(builder.rustdoc(compiler)); } @@ -1557,10 +1662,12 @@ note: if you're sure you want to do this, please open an issue as to why. In the cmd.arg("--coverage-dump-path").arg(coverage_dump); } - if mode == "run-make" || mode == "run-coverage" { + if mode == "coverage-run" { + // The demangler doesn't need the current compiler, so we can avoid + // unnecessary rebuilds by using the bootstrap compiler instead. let rust_demangler = builder .ensure(tool::RustDemangler { - compiler, + compiler: compiler.with_stage(0), target: compiler.host, extra_features: Vec::new(), }) @@ -1678,7 +1785,7 @@ note: if you're sure you want to do this, please open an issue as to why. In the } } - if util::forcing_clang_based_tests() { + if helpers::forcing_clang_based_tests() { let clang_exe = builder.llvm_out(target).join("bin").join("clang"); cmd.arg("--run-clang-based-tests-with").arg(clang_exe); } @@ -1697,7 +1804,7 @@ note: if you're sure you want to do this, please open an issue as to why. In the // Get test-args by striping suite path let mut test_args: Vec<&str> = paths .iter() - .filter_map(|p| util::is_valid_test_suite_arg(p, suite_path, builder)) + .filter_map(|p| helpers::is_valid_test_suite_arg(p, suite_path, builder)) .collect(); test_args.append(&mut builder.config.test_args()); @@ -1747,11 +1854,11 @@ note: if you're sure you want to do this, please open an issue as to why. In the } if !builder.config.dry_run() - && (matches!(suite, "run-make" | "run-make-fulldeps") || mode == "run-coverage") + && (matches!(suite, "run-make" | "run-make-fulldeps") || mode == "coverage-run") { // The llvm/bin directory contains many useful cross-platform // tools. Pass the path to run-make tests so they can use them. - // (The run-coverage tests also need these tools to process + // (The coverage-run tests also need these tools to process // coverage reports.) let llvm_bin_path = llvm_config .parent() @@ -1864,6 +1971,10 @@ note: if you're sure you want to do this, please open an issue as to why. In the cmd.arg("--git-hash"); } + let git_config = builder.config.git_config(); + cmd.arg("--git-repository").arg(git_config.git_repository); + cmd.arg("--nightly-branch").arg(git_config.nightly_branch); + builder.ci_env.force_coloring_in_ci(&mut cmd); #[cfg(feature = "build-metrics")] @@ -1886,7 +1997,7 @@ note: if you're sure you want to do this, please open an issue as to why. In the compiler.host, target, ); - crate::render_tests::try_run_tests(builder, &mut cmd, false); + try_run_tests(builder, &mut cmd, false); if let Some(compare_mode) = compare_mode { cmd.arg("--compare-mode").arg(compare_mode); @@ -1908,8 +2019,8 @@ note: if you're sure you want to do this, please open an issue as to why. In the "Check compiletest suite={} mode={} compare_mode={} ({} -> {})", suite, mode, compare_mode, &compiler.host, target )); - let _time = util::timeit(&builder); - crate::render_tests::try_run_tests(builder, &mut cmd, false); + let _time = helpers::timeit(&builder); + try_run_tests(builder, &mut cmd, false); } } } @@ -1980,7 +2091,7 @@ impl BookTest { compiler.host, compiler.host, ); - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); let toolstate = if builder.run_delaying_failure(&mut rustbook_cmd) { ToolState::TestPass } else { @@ -2002,7 +2113,7 @@ impl BookTest { // Do a breadth-first traversal of the `src/doc` directory and just run // tests for all files that end in `*.md` let mut stack = vec![builder.src.join(self.path)]; - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); let mut files = Vec::new(); while let Some(p) = stack.pop() { if p.is_dir() { @@ -2113,7 +2224,7 @@ impl Step for ErrorIndex { let guard = builder.msg(Kind::Test, compiler.stage, "error-index", compiler.host, compiler.host); - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); builder.run_quiet(&mut tool); drop(guard); // The tests themselves need to link to std, so make sure it is @@ -2235,7 +2346,7 @@ fn run_cargo_test<'a>( ) -> bool { let mut cargo = prepare_cargo_test(cargo, libtest_args, crates, primary_crate, compiler, target, builder); - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); let _group = description.into().and_then(|what| { builder.msg_sysroot_tool(Kind::Test, compiler.stage, what, compiler.host, target) }); @@ -2630,7 +2741,7 @@ impl Step for RemoteCopyLibs { for f in t!(builder.sysroot_libdir(compiler, target).read_dir()) { let f = t!(f); let name = f.file_name().into_string().unwrap(); - if util::is_dylib(&name) { + if helpers::is_dylib(&name) { builder.run(Command::new(&tool).arg("push").arg(f.path())); } } @@ -2675,7 +2786,9 @@ impl Step for Distcheck { .current_dir(&dir), ); builder.run( - Command::new(util::make(&builder.config.build.triple)).arg("check").current_dir(&dir), + Command::new(helpers::make(&builder.config.build.triple)) + .arg("check") + .current_dir(&dir), ); // Now make sure that rust-src has all of libstd's dependencies @@ -2832,7 +2945,7 @@ impl Step for LintDocs { /// Tests that the lint examples in the rustc book generate the correct /// lints and have the expected format. fn run(self, builder: &Builder<'_>) { - builder.ensure(crate::doc::RustcBook { + builder.ensure(crate::core::build_steps::doc::RustcBook { compiler: self.compiler, target: self.target, validate: true, @@ -2940,10 +3053,6 @@ impl Step for TestHelpers { let _guard = builder.msg_unstaged(Kind::Build, "test helpers", target); t!(fs::create_dir_all(&dst)); let mut cfg = cc::Build::new(); - // FIXME: Workaround for https://github.com/emscripten-core/emscripten/issues/9013 - if target.contains("emscripten") { - cfg.pic(false); - } // We may have found various cross-compilers a little differently due to our // extra configuration, so inform cc of these compilers. Note, though, that @@ -2990,15 +3099,7 @@ impl Step for CodegenCranelift { return; } - let triple = run.target.triple; - let target_supported = if triple.contains("linux") { - triple.contains("x86_64") || triple.contains("aarch64") || triple.contains("s390x") - } else if triple.contains("darwin") || triple.contains("windows") { - triple.contains("x86_64") - } else { - false - }; - if !target_supported { + if !target_supports_cranelift_backend(run.target) { builder.info("target not supported by rustc_codegen_cranelift. skipping"); return; } @@ -3055,11 +3156,12 @@ impl Step for CodegenCranelift { &compiler.host, target )); - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); // FIXME handle vendoring for source tarballs before removing the --skip-test below let download_dir = builder.out.join("cg_clif_download"); + // FIXME: Uncomment the `prepare` command below once vendoring is implemented. /* let mut prepare_cargo = build_cargo(); prepare_cargo.arg("--").arg("prepare").arg("--download-dir").arg(&download_dir); @@ -3087,7 +3189,127 @@ impl Step for CodegenCranelift { .arg("testsuite.extended_sysroot"); cargo.args(builder.config.test_args()); + let mut cmd: Command = cargo.into(); + builder.run_cmd(BootstrapCommand::from(&mut cmd).fail_fast()); + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CodegenGCC { + compiler: Compiler, + target: TargetSelection, +} + +impl Step for CodegenGCC { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.paths(&["compiler/rustc_codegen_gcc"]) + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + let host = run.build_triple(); + let compiler = run.builder.compiler_for(run.builder.top_stage, host, host); + + if builder.doc_tests == DocTests::Only { + return; + } + + let triple = run.target.triple; + let target_supported = + if triple.contains("linux") { triple.contains("x86_64") } else { false }; + if !target_supported { + builder.info("target not supported by rustc_codegen_gcc. skipping"); + return; + } + + if builder.remote_tested(run.target) { + builder.info("remote testing is not supported by rustc_codegen_gcc. skipping"); + return; + } + + if !builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("gcc")) { + builder.info("gcc not in rust.codegen-backends. skipping"); + return; + } + + builder.ensure(CodegenGCC { compiler, target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target = self.target; + + builder.ensure(compile::Std::new_with_extra_rust_args( + compiler, + target, + &["-Csymbol-mangling-version=v0", "-Cpanic=abort"], + )); + + // If we're not doing a full bootstrap but we're testing a stage2 + // version of libstd, then what we're actually testing is the libstd + // produced in stage1. Reflect that here by updating the compiler that + // we're working with automatically. + let compiler = builder.compiler_for(compiler.stage, compiler.host, target); + + let build_cargo = || { + let mut cargo = builder.cargo( + compiler, + Mode::Codegen, // Must be codegen to ensure dlopen on compiled dylibs works + SourceType::InTree, + target, + "run", + ); + cargo.current_dir(&builder.src.join("compiler/rustc_codegen_gcc")); + cargo + .arg("--manifest-path") + .arg(builder.src.join("compiler/rustc_codegen_gcc/build_system/Cargo.toml")); + compile::rustc_cargo_env(builder, &mut cargo, target, compiler.stage); + + // Avoid incremental cache issues when changing rustc + cargo.env("CARGO_BUILD_INCREMENTAL", "false"); + cargo.rustflag("-Cpanic=abort"); + + cargo + }; + + builder.info(&format!( + "{} GCC stage{} ({} -> {})", + Kind::Test.description(), + compiler.stage, + &compiler.host, + target + )); + let _time = helpers::timeit(&builder); + + // FIXME: Uncomment the `prepare` command below once vendoring is implemented. + /* + let mut prepare_cargo = build_cargo(); + prepare_cargo.arg("--").arg("prepare"); #[allow(deprecated)] - builder.config.try_run(&mut cargo.into()).unwrap(); + builder.config.try_run(&mut prepare_cargo.into()).unwrap(); + */ + + let mut cargo = build_cargo(); + + cargo + .arg("--") + .arg("test") + .arg("--use-system-gcc") + .arg("--use-backend") + .arg("gcc") + .arg("--out-dir") + .arg(builder.stage_out(compiler, Mode::ToolRustc).join("cg_gcc")) + .arg("--release") + .arg("--no-default-features") + .arg("--mini-tests") + .arg("--std-tests"); + cargo.args(builder.config.test_args()); + + let mut cmd: Command = cargo.into(); + builder.run_cmd(BootstrapCommand::from(&mut cmd).fail_fast()); } } diff --git a/src/bootstrap/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index f094dd9d7..d1bc05e51 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -3,12 +3,13 @@ use std::fs; use std::path::PathBuf; use std::process::Command; -use crate::builder::{Builder, Cargo as CargoCommand, RunConfig, ShouldRun, Step}; -use crate::channel::GitInfo; -use crate::compile; -use crate::config::TargetSelection; -use crate::toolstate::ToolState; -use crate::util::{add_dylib_path, exe, t}; +use crate::core::build_steps::compile; +use crate::core::build_steps::toolstate::ToolState; +use crate::core::builder::{Builder, Cargo as CargoCommand, RunConfig, ShouldRun, Step}; +use crate::core::config::TargetSelection; +use crate::utils::channel::GitInfo; +use crate::utils::exec::BootstrapCommand; +use crate::utils::helpers::{add_dylib_path, exe, t}; use crate::Compiler; use crate::Mode; use crate::{gha, Kind}; @@ -108,8 +109,8 @@ impl Step for ToolBuild { ); let mut cargo = Command::from(cargo); - #[allow(deprecated)] // we check this in `is_optional_tool` in a second - let is_expected = builder.config.try_run(&mut cargo).is_ok(); + // we check this in `is_optional_tool` in a second + let is_expected = builder.run_cmd(BootstrapCommand::from(&mut cargo).allow_failure()); builder.save_toolstate( tool, @@ -421,7 +422,7 @@ impl Step for Rustdoc { fn make_run(run: RunConfig<'_>) { run.builder.ensure(Rustdoc { - // Note: this is somewhat unique in that we actually want a *target* + // NOTE: this is somewhat unique in that we actually want a *target* // compiler here, because rustdoc *is* a compiler. We won't be using // this as the compiler to build with, but rather this is "what // compiler are we producing"? @@ -453,7 +454,7 @@ impl Step for Rustdoc { // compiler, since you do just as much work. if !builder.config.dry_run() && builder.download_rustc() && build_compiler.stage == 0 { println!( - "warning: `download-rustc` does nothing when building stage1 tools; consider using `--stage 2` instead" + "WARNING: `download-rustc` does nothing when building stage1 tools; consider using `--stage 2` instead" ); } @@ -786,9 +787,9 @@ macro_rules! tool_extended { } } -// Note: tools need to be also added to `Builder::get_step_descriptions` in `builder.rs` +// NOTE: tools need to be also added to `Builder::get_step_descriptions` in `builder.rs` // to make `./x.py build <tool>` work. -// Note: Most submodule updates for tools are handled by bootstrap.py, since they're needed just to +// NOTE: Most submodule updates for tools are handled by bootstrap.py, since they're needed just to // invoke Cargo to build bootstrap. See the comment there for more details. tool_extended!((self, builder), Cargofmt, "src/tools/rustfmt", "cargo-fmt", stable=true; diff --git a/src/bootstrap/toolstate.rs b/src/bootstrap/src/core/build_steps/toolstate.rs index 308023537..a451f92b6 100644 --- a/src/bootstrap/toolstate.rs +++ b/src/bootstrap/src/core/build_steps/toolstate.rs @@ -1,5 +1,5 @@ -use crate::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::util::t; +use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::utils::helpers::t; use serde_derive::{Deserialize, Serialize}; use std::collections::HashMap; use std::env; @@ -172,7 +172,7 @@ impl Step for ToolStateCheck { for (tool, _) in STABLE_TOOLS.iter().chain(NIGHTLY_TOOLS.iter()) { if !toolstates.contains_key(*tool) { did_error = true; - eprintln!("error: Tool `{tool}` was not recorded in tool state."); + eprintln!("ERROR: Tool `{tool}` was not recorded in tool state."); } } @@ -190,7 +190,7 @@ impl Step for ToolStateCheck { if state != ToolState::TestPass { if !is_nightly { did_error = true; - eprintln!("error: Tool `{tool}` should be test-pass but is {state}"); + eprintln!("ERROR: Tool `{tool}` should be test-pass but is {state}"); } else if in_beta_week { let old_state = old_toolstate .iter() @@ -200,14 +200,14 @@ impl Step for ToolStateCheck { if state < old_state { did_error = true; eprintln!( - "error: Tool `{tool}` has regressed from {old_state} to {state} during beta week." + "ERROR: Tool `{tool}` has regressed from {old_state} to {state} during beta week." ); } else { // This warning only appears in the logs, which most // people won't read. It's mostly here for testing and // debugging. eprintln!( - "warning: Tool `{tool}` is not test-pass (is `{state}`), \ + "WARNING: Tool `{tool}` is not test-pass (is `{state}`), \ this should be fixed before beta is branched." ); } diff --git a/src/bootstrap/builder.rs b/src/bootstrap/src/core/builder.rs index 46a62eed9..cd276674d 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/src/core/builder.rs @@ -2,7 +2,7 @@ use std::any::{type_name, Any}; use std::cell::{Cell, RefCell}; use std::collections::BTreeSet; use std::env; -use std::ffi::OsStr; +use std::ffi::{OsStr, OsString}; use std::fmt::{Debug, Write}; use std::fs::{self, File}; use std::hash::Hash; @@ -12,20 +12,15 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::time::{Duration, Instant}; -use crate::cache::{Cache, Interned, INTERNER}; -use crate::config::{DryRun, SplitDebuginfo, TargetSelection}; -use crate::doc; -use crate::flags::{Color, Subcommand}; -use crate::install; -use crate::llvm; -use crate::run; -use crate::setup; -use crate::test; -use crate::tool::{self, SourceType}; -use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir, output, t}; +use crate::core::build_steps::llvm; +use crate::core::build_steps::tool::{self, SourceType}; +use crate::core::build_steps::{check, clean, compile, dist, doc, install, run, setup, test}; +use crate::core::config::flags::{Color, Subcommand}; +use crate::core::config::{DryRun, SplitDebuginfo, TargetSelection}; +use crate::utils::cache::{Cache, Interned, INTERNER}; +use crate::utils::helpers::{self, add_dylib_path, add_link_lib_path, exe, libdir, output, t}; +use crate::Crate; use crate::EXTRA_CHECK_CFGS; -use crate::{check, compile, Crate}; -use crate::{clean, dist}; use crate::{Build, CLang, DocTests, GitRepo, Mode}; pub use crate::Compiler; @@ -36,6 +31,10 @@ pub use crate::Compiler; use clap::ValueEnum; use once_cell::sync::{Lazy, OnceCell}; +#[cfg(test)] +#[path = "../tests/builder.rs"] +mod tests; + pub struct Builder<'a> { pub build: &'a Build, pub top_stage: u32, @@ -387,13 +386,13 @@ impl StepDescription { } if !paths.is_empty() { - eprintln!("error: no `{}` rules matched {:?}", builder.kind.as_str(), paths,); + eprintln!("ERROR: no `{}` rules matched {:?}", builder.kind.as_str(), paths,); eprintln!( - "help: run `x.py {} --help --verbose` to show a list of available paths", + "HELP: run `x.py {} --help --verbose` to show a list of available paths", builder.kind.as_str() ); eprintln!( - "note: if you are adding a new Step to bootstrap itself, make sure you register it with `describe!`" + "NOTE: if you are adding a new Step to bootstrap itself, make sure you register it with `describe!`" ); crate::exit!(1); } @@ -723,13 +722,14 @@ impl<'a> Builder<'a> { check::Bootstrap ), Kind::Test => describe!( - crate::toolstate::ToolStateCheck, + crate::core::build_steps::toolstate::ToolStateCheck, test::ExpandYamlAnchors, test::Tidy, test::Ui, test::RunPassValgrind, + test::Coverage, test::CoverageMap, - test::RunCoverage, + test::CoverageRun, test::MirOpt, test::Codegen, test::CodegenUnits, @@ -738,8 +738,9 @@ impl<'a> Builder<'a> { test::Debuginfo, test::UiFullDeps, test::CodegenCranelift, + test::CodegenGCC, test::Rustdoc, - test::RunCoverageRustdoc, + test::CoverageRunRustdoc, test::Pretty, test::Crate, test::CrateLibrustc, @@ -816,6 +817,7 @@ impl<'a> Builder<'a> { dist::JsonDocs, dist::Mingw, dist::Rustc, + dist::CodegenBackend, dist::Std, dist::RustcDev, dist::Analysis, @@ -1174,9 +1176,6 @@ impl<'a> Builder<'a> { if let Some(linker) = self.linker(compiler.host) { cmd.env("RUSTDOC_LINKER", linker); } - if self.is_fuse_ld_lld(compiler.host) { - cmd.env("RUSTDOC_FUSE_LD_LLD", "1"); - } cmd } @@ -1268,6 +1267,8 @@ impl<'a> Builder<'a> { let mut cargo = self.bare_cargo(compiler, mode, target, cmd); let out_dir = self.stage_out(compiler, mode); + let mut hostflags = HostFlags::default(); + // Codegen backends are not yet tracked by -Zbinary-dep-depinfo, // so we need to explicitly clear out if they've been updated. for backend in self.codegen_backends(compiler) { @@ -1298,8 +1299,8 @@ impl<'a> Builder<'a> { // See comment in rustc_llvm/build.rs for why this is necessary, largely llvm-config // needs to not accidentally link to libLLVM in stage0/lib. - cargo.env("REAL_LIBRARY_PATH_VAR", &util::dylib_path_var()); - if let Some(e) = env::var_os(util::dylib_path_var()) { + cargo.env("REAL_LIBRARY_PATH_VAR", &helpers::dylib_path_var()); + if let Some(e) = env::var_os(helpers::dylib_path_var()) { cargo.env("REAL_LIBRARY_PATH", e); } @@ -1312,7 +1313,7 @@ impl<'a> Builder<'a> { // rustc_llvm. But if LLVM is stale, that'll be a tiny amount // of work comparatively, and we'd likely need to rebuild it anyway, // so that's okay. - if crate::llvm::prebuilt_llvm_config(self, target).is_err() { + if crate::core::build_steps::llvm::prebuilt_llvm_config(self, target).is_err() { cargo.env("RUST_CHECK", "1"); } } @@ -1359,9 +1360,9 @@ impl<'a> Builder<'a> { } }).unwrap_or_else(|_| { eprintln!( - "error: `x.py clippy` requires a host `rustc` toolchain with the `clippy` component" + "ERROR: `x.py clippy` requires a host `rustc` toolchain with the `clippy` component" ); - eprintln!("help: try `rustup component add clippy`"); + eprintln!("HELP: try `rustup component add clippy`"); crate::exit!(1); }); if !t!(std::str::from_utf8(&output.stdout)).contains("nightly") { @@ -1403,19 +1404,28 @@ impl<'a> Builder<'a> { rustflags.arg("-Zunstable-options"); } - // Enable cfg checking of cargo features for everything but std and also enable cfg - // checking of names and values. + // #[cfg(bootstrap)] + let use_new_check_cfg_syntax = self.local_rebuild; + + // Enable compile-time checking of `cfg` names, values and Cargo `features`. // // Note: `std`, `alloc` and `core` imports some dependencies by #[path] (like // backtrace, core_simd, std_float, ...), those dependencies have their own // features but cargo isn't involved in the #[path] process and so cannot pass the // complete list of features, so for that reason we don't enable checking of // features for std crates. - cargo.arg(if mode != Mode::Std { - "-Zcheck-cfg=names,values,output,features" + if use_new_check_cfg_syntax { + cargo.arg("-Zcheck-cfg"); + if mode == Mode::Std { + rustflags.arg("--check-cfg=cfg(feature,values(any()))"); + } } else { - "-Zcheck-cfg=names,values,output" - }); + cargo.arg(if mode != Mode::Std { + "-Zcheck-cfg=names,values,output,features" + } else { + "-Zcheck-cfg=names,values,output" + }); + } // Add extra cfg not defined in/by rustc // @@ -1435,10 +1445,33 @@ impl<'a> Builder<'a> { .collect::<String>(), None => String::new(), }; - rustflags.arg(&format!("--check-cfg=values({name}{values})")); + if use_new_check_cfg_syntax { + let values = values.strip_prefix(",").unwrap_or(&values); // remove the first `,` + rustflags.arg(&format!("--check-cfg=cfg({name},values({values}))")); + } else { + rustflags.arg(&format!("--check-cfg=values({name}{values})")); + } } } + // FIXME(rust-lang/cargo#5754) we shouldn't be using special command arguments + // to the host invocation here, but rather Cargo should know what flags to pass rustc + // itself. + if stage == 0 { + hostflags.arg("--cfg=bootstrap"); + } + // Cargo doesn't pass RUSTFLAGS to proc_macros: + // https://github.com/rust-lang/cargo/issues/4423 + // Thus, if we are on stage 0, we explicitly set `--cfg=bootstrap`. + // We also declare that the flag is expected, which we need to do to not + // get warnings about it being unexpected. + hostflags.arg("-Zunstable-options"); + if use_new_check_cfg_syntax { + hostflags.arg("--check-cfg=cfg(bootstrap)"); + } else { + hostflags.arg("--check-cfg=values(bootstrap)"); + } + // FIXME: It might be better to use the same value for both `RUSTFLAGS` and `RUSTDOCFLAGS`, // but this breaks CI. At the very least, stage0 `rustdoc` needs `--cfg bootstrap`. See // #71458. @@ -1630,7 +1663,7 @@ impl<'a> Builder<'a> { // argument manually via `-C link-args=-Wl,-rpath,...`. Plus isn't it // fun to pass a flag to a tool to pass a flag to pass a flag to a tool // to change a flag in a binary? - if self.config.rpath_enabled(target) && util::use_host_linker(target) { + if self.config.rpath_enabled(target) && helpers::use_host_linker(target) { let libdir = self.sysroot_libdir_relative(compiler).to_str().unwrap(); let rpath = if target.contains("apple") { // Note that we need to take one extra step on macOS to also pass @@ -1655,11 +1688,10 @@ impl<'a> Builder<'a> { } if let Some(host_linker) = self.linker(compiler.host) { - cargo.env("RUSTC_HOST_LINKER", host_linker); + hostflags.arg(format!("-Clinker={}", host_linker.display())); } if self.is_fuse_ld_lld(compiler.host) { - cargo.env("RUSTC_HOST_FUSE_LD_LLD", "1"); - cargo.env("RUSTDOC_FUSE_LD_LLD", "1"); + hostflags.arg("-Clink-args=-fuse-ld=lld"); } if let Some(target_linker) = self.linker(target) { @@ -1743,7 +1775,8 @@ impl<'a> Builder<'a> { } if let Some(x) = self.crt_static(compiler.host) { - cargo.env("RUSTC_HOST_CRT_STATIC", x.to_string()); + let sign = if x { "+" } else { "-" }; + hostflags.arg(format!("-Ctarget-feature={sign}crt-static")); } if let Some(map_to) = self.build.debuginfo_map_to(GitRepo::Rustc) { @@ -1755,6 +1788,20 @@ impl<'a> Builder<'a> { cargo.env("CFG_VIRTUAL_RUST_SOURCE_BASE_DIR", map_to); } + if self.config.rust_remap_debuginfo { + // FIXME: handle vendored sources + let registry_src = t!(home::cargo_home()).join("registry").join("src"); + let mut env_var = OsString::new(); + for entry in t!(std::fs::read_dir(registry_src)) { + if !env_var.is_empty() { + env_var.push("\t"); + } + env_var.push(t!(entry).path()); + env_var.push("=/rust/deps"); + } + cargo.env("RUSTC_CARGO_REGISTRY_SRC_TO_REMAP", env_var); + } + // Enable usage of unstable features cargo.env("RUSTC_BOOTSTRAP", "1"); self.add_rust_test_threads(&mut cargo); @@ -2055,7 +2102,7 @@ impl<'a> Builder<'a> { cargo.env("RUSTFLAGS", &rustc_args.join(" ")); } - Cargo { command: cargo, rustflags, rustdocflags, allow_features } + Cargo { command: cargo, rustflags, rustdocflags, hostflags, allow_features } } /// Ensure that a given step is built, returning its output. This will @@ -2184,9 +2231,6 @@ impl<'a> Builder<'a> { } } -#[cfg(test)] -mod tests; - /// Represents flag values in `String` form with whitespace delimiter to pass it to the compiler later. /// /// `-Z crate-attr` flags will be applied recursively on the target code using the `rustc_parse::parser::Parser`. @@ -2233,11 +2277,36 @@ impl Rustflags { } } +/// Flags that are passed to the `rustc` shim binary. +/// These flags will only be applied when compiling host code, i.e. when +/// `--target` is unset. +#[derive(Debug, Default)] +pub struct HostFlags { + rustc: Vec<String>, +} + +impl HostFlags { + const SEPARATOR: &'static str = " "; + + /// Adds a host rustc flag. + fn arg<S: Into<String>>(&mut self, flag: S) { + let value = flag.into().trim().to_string(); + assert!(!value.contains(Self::SEPARATOR)); + self.rustc.push(value); + } + + /// Encodes all the flags into a single string. + fn encode(self) -> String { + self.rustc.join(Self::SEPARATOR) + } +} + #[derive(Debug)] pub struct Cargo { command: Command, rustflags: Rustflags, rustdocflags: Rustflags, + hostflags: HostFlags, allow_features: String, } @@ -2309,6 +2378,11 @@ impl From<Cargo> for Command { cargo.command.env("RUSTDOCFLAGS", rustdocflags); } + let encoded_hostflags = cargo.hostflags.encode(); + if !encoded_hostflags.is_empty() { + cargo.command.env("RUSTC_HOST_FLAGS", encoded_hostflags); + } + if !cargo.allow_features.is_empty() { cargo.command.env("RUSTC_ALLOW_FEATURES", cargo.allow_features); } diff --git a/src/bootstrap/config.rs b/src/bootstrap/src/core/config/config.rs index 836328f94..0a9175aa3 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -4,6 +4,7 @@ //! how the build runs. #[cfg(test)] +#[path = "../../tests/config.rs"] mod tests; use std::cell::{Cell, RefCell}; @@ -17,19 +18,21 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::str::FromStr; -use crate::cache::{Interned, INTERNER}; -use crate::cc_detect::{ndk_compiler, Language}; -use crate::channel::{self, GitInfo}; -use crate::compile::CODEGEN_BACKEND_PREFIX; -pub use crate::flags::Subcommand; -use crate::flags::{Color, Flags, Warnings}; -use crate::util::{exe, output, t}; +use crate::core::build_steps::compile::CODEGEN_BACKEND_PREFIX; +use crate::core::build_steps::llvm; +use crate::core::config::flags::{Color, Flags, Warnings}; +use crate::utils::cache::{Interned, INTERNER}; +use crate::utils::channel::{self, GitInfo}; +use crate::utils::helpers::{exe, output, t}; use build_helper::exit; use once_cell::sync::OnceCell; use semver::Version; use serde::{Deserialize, Deserializer}; use serde_derive::Deserialize; +pub use crate::core::config::flags::Subcommand; +use build_helper::git::GitConfig; + macro_rules! check_ci_llvm { ($name:expr) => { assert!( @@ -112,7 +115,8 @@ impl Display for DebuginfoLevel { /// `config.example.toml`. #[derive(Default, Clone)] pub struct Config { - pub changelog_seen: Option<usize>, + pub changelog_seen: Option<usize>, // FIXME: Deprecated field. Remove it at 2024. + pub change_id: Option<usize>, pub ccache: Option<String>, /// Call Build::ninja() instead of this. pub ninja_in_file: bool, @@ -139,6 +143,7 @@ pub struct Config { pub color: Color, pub patch_binaries_for_nix: Option<bool>, pub stage0_metadata: Stage0Metadata, + pub android_ndk: Option<PathBuf>, pub stdout_is_tty: bool, pub stderr_is_tty: bool, @@ -233,6 +238,7 @@ pub struct Config { pub llvm_profile_use: Option<String>, pub llvm_profile_generate: bool, pub llvm_libunwind_default: Option<LlvmLibunwind>, + pub enable_bolt_settings: bool, pub reproducible_artifacts: Vec<String>, @@ -314,6 +320,7 @@ pub struct Stage0Config { pub artifacts_server: String, pub artifacts_with_llvm_assertions_server: String, pub git_merge_commit_email: String, + pub git_repository: String, pub nightly_branch: String, } #[derive(Default, Deserialize, Clone)] @@ -517,7 +524,6 @@ pub struct Target { pub ranlib: Option<PathBuf>, pub default_linker: Option<PathBuf>, pub linker: Option<PathBuf>, - pub ndk: Option<PathBuf>, pub sanitizers: Option<bool>, pub profiler: Option<StringOrBool>, pub rpath: Option<bool>, @@ -545,8 +551,9 @@ impl Target { /// `Config` structure. #[derive(Deserialize, Default)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] -struct TomlConfig { - changelog_seen: Option<usize>, +pub(crate) struct TomlConfig { + changelog_seen: Option<usize>, // FIXME: Deprecated field. Remove it at 2024. + change_id: Option<usize>, build: Option<Build>, install: Option<Install>, llvm: Option<Llvm>, @@ -574,7 +581,17 @@ trait Merge { impl Merge for TomlConfig { fn merge( &mut self, - TomlConfig { build, install, llvm, rust, dist, target, profile: _, changelog_seen }: Self, + TomlConfig { + build, + install, + llvm, + rust, + dist, + target, + profile: _, + changelog_seen, + change_id, + }: Self, replace: ReplaceOpt, ) { fn do_merge<T: Merge>(x: &mut Option<T>, y: Option<T>, replace: ReplaceOpt) { @@ -587,6 +604,7 @@ impl Merge for TomlConfig { } } self.changelog_seen.merge(changelog_seen, replace); + self.change_id.merge(change_id, replace); do_merge(&mut self.build, build, replace); do_merge(&mut self.install, install, replace); do_merge(&mut self.llvm, llvm, replace); @@ -783,6 +801,7 @@ define_config! { patch_binaries_for_nix: Option<bool> = "patch-binaries-for-nix", // NOTE: only parsed by bootstrap.py, `--feature build-metrics` enables metrics unconditionally metrics: Option<bool> = "metrics", + android_ndk: Option<PathBuf> = "android-ndk", } } @@ -1023,7 +1042,6 @@ define_config! { llvm_has_rust_patches: Option<bool> = "llvm-has-rust-patches", llvm_filecheck: Option<String> = "llvm-filecheck", llvm_libunwind: Option<String> = "llvm-libunwind", - android_ndk: Option<String> = "android-ndk", sanitizers: Option<bool> = "sanitizers", profiler: Option<StringOrBool> = "profiler", rpath: Option<bool> = "rpath", @@ -1057,6 +1075,7 @@ impl Config { config.bindir = "bin".into(); config.dist_include_mingw_linker = true; config.dist_compression_profile = "fast".into(); + config.rustc_parallel = true; config.stdout_is_tty = std::io::stdout().is_terminal(); config.stderr_is_tty = std::io::stderr().is_terminal(); @@ -1115,6 +1134,7 @@ impl Config { config.free_args = std::mem::take(&mut flags.free_args); config.llvm_profile_use = flags.llvm_profile_use; config.llvm_profile_generate = flags.llvm_profile_generate; + config.enable_bolt_settings = flags.enable_bolt_settings; // Infer the rest of the configuration. @@ -1242,6 +1262,7 @@ impl Config { toml.merge(override_toml, ReplaceOpt::Override); config.changelog_seen = toml.changelog_seen; + config.change_id = toml.change_id; let build = toml.build.unwrap_or_default(); if let Some(file_build) = build.build { @@ -1253,12 +1274,13 @@ impl Config { // To avoid writing to random places on the file system, `config.out` needs to be an absolute path. if !config.out.is_absolute() { // `canonicalize` requires the path to already exist. Use our vendored copy of `absolute` instead. - config.out = crate::util::absolute(&config.out); + config.out = crate::utils::helpers::absolute(&config.out); } config.initial_rustc = if let Some(rustc) = build.rustc { - // FIXME(#115065): re-enable this check - // config.check_build_rustc_version(&rustc); + if !flags.skip_stage0_validation { + config.check_build_rustc_version(&rustc); + } PathBuf::from(rustc) } else { config.download_beta_toolchain(); @@ -1302,6 +1324,7 @@ impl Config { config.python = build.python.map(PathBuf::from); config.reuse = build.reuse.map(PathBuf::from); config.submodules = build.submodules; + config.android_ndk = build.android_ndk; set(&mut config.low_priority, build.low_priority); set(&mut config.compiler_docs, build.compiler_docs); set(&mut config.library_docs_private_items, build.library_docs_private_items); @@ -1410,7 +1433,9 @@ impl Config { set(&mut config.use_lld, rust.use_lld); set(&mut config.lld_enabled, rust.lld); set(&mut config.llvm_tools_enabled, rust.llvm_tools); - config.rustc_parallel = rust.parallel_compiler.unwrap_or(false); + config.rustc_parallel = rust + .parallel_compiler + .unwrap_or(config.channel == "dev" || config.channel == "nightly"); config.rustc_default_linker = rust.default_linker; config.musl_root = rust.musl_root.map(PathBuf::from); config.save_toolstates = rust.save_toolstates.map(PathBuf::from); @@ -1439,7 +1464,7 @@ impl Config { if available_backends.contains(&backend) { panic!("Invalid value '{s}' for 'rust.codegen-backends'. Instead, please use '{backend}'."); } else { - println!("help: '{s}' for 'rust.codegen-backends' might fail. \ + println!("HELP: '{s}' for 'rust.codegen-backends' might fail. \ Codegen backends are mostly defined without the '{CODEGEN_BACKEND_PREFIX}' prefix. \ In this case, it would be referred to as '{backend}'."); } @@ -1508,16 +1533,7 @@ impl Config { config.llvm_build_config = llvm.build_config.clone().unwrap_or(Default::default()); let asserts = llvm_assertions.unwrap_or(false); - config.llvm_from_ci = match llvm.download_ci_llvm { - Some(StringOrBool::String(s)) => { - assert_eq!(s, "if-available", "unknown option `{s}` for download-ci-llvm"); - crate::llvm::is_ci_llvm_available(&config, asserts) - } - Some(StringOrBool::Bool(b)) => b, - None => { - config.channel == "dev" && crate::llvm::is_ci_llvm_available(&config, asserts) - } - }; + config.llvm_from_ci = config.parse_download_ci_llvm(llvm.download_ci_llvm, asserts); if config.llvm_from_ci { // None of the LLVM options, except assertions, are supported @@ -1557,8 +1573,8 @@ impl Config { config.llvm_link_shared.set(Some(true)); } } else { - config.llvm_from_ci = - config.channel == "dev" && crate::llvm::is_ci_llvm_available(&config, false); + config.llvm_from_ci = config.channel == "dev" + && crate::core::build_steps::llvm::is_ci_llvm_available(&config, false); } if let Some(t) = toml.target { @@ -1581,18 +1597,11 @@ impl Config { .llvm_libunwind .as_ref() .map(|v| v.parse().expect("failed to parse rust.llvm-libunwind")); - if let Some(ref s) = cfg.android_ndk { - target.ndk = Some(config.src.join(s)); - } if let Some(s) = cfg.no_std { target.no_std = s; } - target.cc = cfg.cc.map(PathBuf::from).or_else(|| { - target.ndk.as_ref().map(|ndk| ndk_compiler(Language::C, &triple, ndk)) - }); - target.cxx = cfg.cxx.map(PathBuf::from).or_else(|| { - target.ndk.as_ref().map(|ndk| ndk_compiler(Language::CPlusPlus, &triple, ndk)) - }); + target.cc = cfg.cc.map(PathBuf::from); + target.cxx = cfg.cxx.map(PathBuf::from); target.ar = cfg.ar.map(PathBuf::from); target.ranlib = cfg.ranlib.map(PathBuf::from); target.linker = cfg.linker.map(PathBuf::from); @@ -1799,9 +1808,9 @@ impl Config { } (channel, version) => { let src = self.src.display(); - eprintln!("error: failed to determine artifact channel and/or version"); + eprintln!("ERROR: failed to determine artifact channel and/or version"); eprintln!( - "help: consider using a git checkout or ensure these files are readable" + "HELP: consider using a git checkout or ensure these files are readable" ); if let Err(channel) = channel { eprintln!("reading {src}/src/ci/channel failed: {channel:?}"); @@ -1993,6 +2002,13 @@ impl Config { self.rust_codegen_backends.get(0).cloned() } + pub fn git_config(&self) -> GitConfig<'_> { + GitConfig { + git_repository: &self.stage0_metadata.config.git_repository, + nightly_branch: &self.stage0_metadata.config.nightly_branch, + } + } + pub fn check_build_rustc_version(&self, rustc_path: &str) { if self.dry_run() { return; @@ -2057,10 +2073,10 @@ impl Config { ); let commit = merge_base.trim_end(); if commit.is_empty() { - println!("error: could not find commit hash for downloading rustc"); - println!("help: maybe your repository history is too shallow?"); - println!("help: consider disabling `download-rustc`"); - println!("help: or fetch enough history to include one upstream commit"); + println!("ERROR: could not find commit hash for downloading rustc"); + println!("HELP: maybe your repository history is too shallow?"); + println!("HELP: consider disabling `download-rustc`"); + println!("HELP: or fetch enough history to include one upstream commit"); crate::exit!(1); } @@ -2074,20 +2090,108 @@ impl Config { if if_unchanged { if self.verbose > 0 { println!( - "warning: saw changes to compiler/ or library/ since {commit}; \ + "WARNING: saw changes to compiler/ or library/ since {commit}; \ ignoring `download-rustc`" ); } return None; } println!( - "warning: `download-rustc` is enabled, but there are changes to \ + "WARNING: `download-rustc` is enabled, but there are changes to \ compiler/ or library/" ); } Some(commit.to_string()) } + + fn parse_download_ci_llvm( + &self, + download_ci_llvm: Option<StringOrBool>, + asserts: bool, + ) -> bool { + match download_ci_llvm { + None => self.channel == "dev" && llvm::is_ci_llvm_available(&self, asserts), + Some(StringOrBool::Bool(b)) => b, + Some(StringOrBool::String(s)) if s == "if-available" => { + llvm::is_ci_llvm_available(&self, asserts) + } + Some(StringOrBool::String(s)) if s == "if-unchanged" => { + // Git is needed to track modifications here, but tarball source is not available. + // If not modified here or built through tarball source, we maintain consistency + // with '"if available"'. + if !self.rust_info.is_from_tarball() + && self + .last_modified_commit(&["src/llvm-project"], "download-ci-llvm", true) + .is_none() + { + // there are some untracked changes in the the given paths. + false + } else { + llvm::is_ci_llvm_available(&self, asserts) + } + } + Some(StringOrBool::String(other)) => { + panic!("unrecognized option for download-ci-llvm: {:?}", other) + } + } + } + + /// Returns the last commit in which any of `modified_paths` were changed, + /// or `None` if there are untracked changes in the working directory and `if_unchanged` is true. + pub fn last_modified_commit( + &self, + modified_paths: &[&str], + option_name: &str, + if_unchanged: bool, + ) -> Option<String> { + // Handle running from a directory other than the top level + let top_level = output(self.git().args(&["rev-parse", "--show-toplevel"])); + let top_level = top_level.trim_end(); + + // Look for a version to compare to based on the current commit. + // Only commits merged by bors will have CI artifacts. + let merge_base = output( + self.git() + .arg("rev-list") + .arg(format!("--author={}", self.stage0_metadata.config.git_merge_commit_email)) + .args(&["-n1", "--first-parent", "HEAD"]), + ); + let commit = merge_base.trim_end(); + if commit.is_empty() { + println!("error: could not find commit hash for downloading components from CI"); + println!("help: maybe your repository history is too shallow?"); + println!("help: consider disabling `{option_name}`"); + println!("help: or fetch enough history to include one upstream commit"); + crate::exit!(1); + } + + // Warn if there were changes to the compiler or standard library since the ancestor commit. + let mut git = self.git(); + git.args(&["diff-index", "--quiet", &commit, "--"]); + + for path in modified_paths { + git.arg(format!("{top_level}/{path}")); + } + + let has_changes = !t!(git.status()).success(); + if has_changes { + if if_unchanged { + if self.verbose > 0 { + println!( + "warning: saw changes to one of {modified_paths:?} since {commit}; \ + ignoring `{option_name}`" + ); + } + return None; + } + println!( + "warning: `{option_name}` is enabled, but there are changes to one of {modified_paths:?}" + ); + } + + Some(commit.to_string()) + } } fn set<T>(field: &mut T, val: Option<T>) { diff --git a/src/bootstrap/flags.rs b/src/bootstrap/src/core/config/flags.rs index e0291e407..2a301007a 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/src/core/config/flags.rs @@ -7,9 +7,9 @@ use std::path::{Path, PathBuf}; use clap::{CommandFactory, Parser, ValueEnum}; -use crate::builder::{Builder, Kind}; -use crate::config::{target_selection_list, Config, TargetSelectionList}; -use crate::setup::Profile; +use crate::core::build_steps::setup::Profile; +use crate::core::builder::{Builder, Kind}; +use crate::core::config::{target_selection_list, Config, TargetSelectionList}; use crate::{Build, DocTests}; #[derive(Copy, Clone, Default, Debug, ValueEnum)] @@ -152,6 +152,12 @@ pub struct Flags { /// generate PGO profile with llvm built for rustc #[arg(global(true), long)] pub llvm_profile_generate: bool, + /// Enable BOLT link flags + #[arg(global(true), long)] + pub enable_bolt_settings: bool, + /// Skip stage0 compiler validation + #[arg(global(true), long)] + pub skip_stage0_validation: bool, /// Additional reproducible artifacts that should be added to the reproducible artifacts archive. #[arg(global(true), long)] pub reproducible_artifact: Vec<String>, @@ -184,7 +190,7 @@ impl Flags { if let Ok(HelpVerboseOnly { help: true, verbose: 1.., cmd: subcommand }) = HelpVerboseOnly::try_parse_from(it.clone()) { - println!("note: updating submodules before printing available paths"); + println!("NOTE: updating submodules before printing available paths"); let config = Config::parse(&[String::from("build")]); let build = Build::new(config); let paths = Builder::get_help(&build, subcommand); diff --git a/src/bootstrap/src/core/config/mod.rs b/src/bootstrap/src/core/config/mod.rs new file mode 100644 index 000000000..9c6861826 --- /dev/null +++ b/src/bootstrap/src/core/config/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod config; +pub(crate) mod flags; + +pub use config::*; diff --git a/src/bootstrap/download.rs b/src/bootstrap/src/core/download.rs index 8e9614ec8..3327aed96 100644 --- a/src/bootstrap/download.rs +++ b/src/bootstrap/src/core/download.rs @@ -11,13 +11,10 @@ use build_helper::ci::CiEnv; use once_cell::sync::OnceCell; use xz2::bufread::XzDecoder; -use crate::{ - config::RustfmtMetadata, - llvm::detect_llvm_sha, - t, - util::{check_run, exe, program_out_of_date}, - Config, -}; +use crate::core::build_steps::llvm::detect_llvm_sha; +use crate::core::config::RustfmtMetadata; +use crate::utils::helpers::{check_run, exe, program_out_of_date}; +use crate::{t, Config}; static SHOULD_FIX_BINS_AND_DYLIBS: OnceCell<bool> = OnceCell::new(); @@ -117,7 +114,7 @@ impl Config { is_nixos }); if val { - eprintln!("info: You seem to be using Nix."); + eprintln!("INFO: You seem to be using Nix."); } val } @@ -320,25 +317,43 @@ impl Config { } /// Returns whether the SHA256 checksum of `path` matches `expected`. - fn verify(&self, path: &Path, expected: &str) -> bool { + pub(crate) fn verify(&self, path: &Path, expected: &str) -> bool { use sha2::Digest; self.verbose(&format!("verifying {}", path.display())); + + if self.dry_run() { + return false; + } + let mut hasher = sha2::Sha256::new(); - // FIXME: this is ok for rustfmt (4.1 MB large at time of writing), but it seems memory-intensive for rustc and larger components. - // Consider using streaming IO instead? - let contents = if self.dry_run() { vec![] } else { t!(fs::read(path)) }; - hasher.update(&contents); - let found = hex::encode(hasher.finalize().as_slice()); - let verified = found == expected; - if !verified && !self.dry_run() { + + let file = t!(File::open(path)); + let mut reader = BufReader::new(file); + + loop { + let buffer = t!(reader.fill_buf()); + let l = buffer.len(); + // break if EOF + if l == 0 { + break; + } + hasher.update(buffer); + reader.consume(l); + } + + let checksum = hex::encode(hasher.finalize().as_slice()); + let verified = checksum == expected; + + if !verified { println!( "invalid checksum: \n\ - found: {found}\n\ + found: {checksum}\n\ expected: {expected}", ); } - return verified; + + verified } } @@ -591,10 +606,10 @@ impl Config { let mut help_on_error = ""; if destination == "ci-rustc" { - help_on_error = "error: failed to download pre-built rustc from CI + help_on_error = "ERROR: failed to download pre-built rustc from CI -note: old builds get deleted after a certain time -help: if trying to compile an old commit of rustc, disable `download-rustc` in config.toml: +NOTE: old builds get deleted after a certain time +HELP: if trying to compile an old commit of rustc, disable `download-rustc` in config.toml: [rust] download-rustc = false @@ -670,10 +685,10 @@ download-rustc = false let filename = format!("rust-dev-{}-{}.tar.xz", version, self.build.triple); let tarball = rustc_cache.join(&filename); if !tarball.exists() { - let help_on_error = "error: failed to download llvm from ci + let help_on_error = "ERROR: failed to download llvm from ci - help: old builds get deleted after a certain time - help: if trying to compile an old commit of rustc, disable `download-ci-llvm` in config.toml: + HELP: old builds get deleted after a certain time + HELP: if trying to compile an old commit of rustc, disable `download-ci-llvm` in config.toml: [llvm] download-ci-llvm = false diff --git a/src/bootstrap/metadata.rs b/src/bootstrap/src/core/metadata.rs index 3b20ceac8..580208232 100644 --- a/src/bootstrap/metadata.rs +++ b/src/bootstrap/src/core/metadata.rs @@ -3,8 +3,8 @@ use std::process::Command; use serde_derive::Deserialize; -use crate::cache::INTERNER; -use crate::util::output; +use crate::utils::cache::INTERNER; +use crate::utils::helpers::output; use crate::{t, Build, Crate}; /// For more information, see the output of diff --git a/src/bootstrap/src/core/mod.rs b/src/bootstrap/src/core/mod.rs new file mode 100644 index 000000000..9e18d6704 --- /dev/null +++ b/src/bootstrap/src/core/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod build_steps; +pub(crate) mod builder; +pub(crate) mod config; +pub(crate) mod download; +pub(crate) mod metadata; +pub(crate) mod sanity; diff --git a/src/bootstrap/sanity.rs b/src/bootstrap/src/core/sanity.rs index 0febdf250..eec3be66a 100644 --- a/src/bootstrap/sanity.rs +++ b/src/bootstrap/src/core/sanity.rs @@ -15,9 +15,9 @@ use std::fs; use std::path::PathBuf; use std::process::Command; -use crate::cache::INTERNER; -use crate::config::Target; -use crate::util::output; +use crate::core::config::Target; +use crate::utils::cache::INTERNER; +use crate::utils::helpers::output; use crate::Build; pub struct Finder { diff --git a/src/bootstrap/lib.rs b/src/bootstrap/src/lib.rs index 8b8d4b237..33b8f1a7c 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -23,75 +23,32 @@ use std::fmt::Display; use std::fs::{self, File}; use std::io; use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; +use std::process::{Command, Output, Stdio}; use std::str; use build_helper::ci::{gha, CiEnv}; use build_helper::exit; -use channel::GitInfo; -use config::{DryRun, Target}; +use build_helper::util::fail; use filetime::FileTime; use once_cell::sync::OnceCell; +use termcolor::{ColorChoice, StandardStream, WriteColor}; +use utils::channel::GitInfo; -use crate::builder::Kind; -use crate::config::{LlvmLibunwind, TargetSelection}; -use crate::util::{ - dir_is_empty, exe, libdir, mtime, output, run, run_suppressed, symlink_dir, try_run_suppressed, -}; - -mod builder; -mod cache; -mod cc_detect; -mod channel; -mod check; -mod clean; -mod compile; -mod config; -mod dist; -mod doc; -mod download; -mod flags; -mod format; -mod install; -mod llvm; -mod metadata; -mod render_tests; -mod run; -mod sanity; -mod setup; -mod suggest; -mod synthetic_targets; -mod tarball; -mod test; -mod tool; -mod toolstate; -pub mod util; - -#[cfg(feature = "build-metrics")] -mod metrics; - -#[cfg(windows)] -mod job; - -#[cfg(all(unix, not(target_os = "haiku")))] -mod job { - pub unsafe fn setup(build: &mut crate::Build) { - if build.config.low_priority { - libc::setpriority(libc::PRIO_PGRP as _, 0, 10); - } - } -} +use crate::core::builder; +use crate::core::builder::Kind; +use crate::core::config::flags; +use crate::core::config::{DryRun, Target}; +use crate::core::config::{LlvmLibunwind, TargetSelection}; +use crate::utils::cache::{Interned, INTERNER}; +use crate::utils::exec::{BehaviorOnFailure, BootstrapCommand, OutputMode}; +use crate::utils::helpers::{self, dir_is_empty, exe, libdir, mtime, output, symlink_dir}; -#[cfg(any(target_os = "haiku", target_os = "hermit", not(any(unix, windows))))] -mod job { - pub unsafe fn setup(_build: &mut crate::Build) {} -} +mod core; +mod utils; -pub use crate::builder::PathSet; -use crate::cache::{Interned, INTERNER}; -pub use crate::config::Config; -pub use crate::flags::Subcommand; -use termcolor::{ColorChoice, StandardStream, WriteColor}; +pub use crate::core::builder::PathSet; +pub use crate::core::config::flags::Subcommand; +pub use crate::core::config::Config; const LLVM_TOOLS: &[&str] = &[ "llvm-cov", // used to generate coverage report @@ -112,7 +69,16 @@ const LLVM_TOOLS: &[&str] = &[ /// LLD file names for all flavors. const LLD_FILE_NAMES: &[&str] = &["ld.lld", "ld64.lld", "lld-link", "wasm-ld"]; -pub const VERSION: usize = 2; +/// Keeps track of major changes made to the bootstrap configuration. +/// +/// These values also represent the IDs of the PRs that caused major changes. +/// You can visit `https://github.com/rust-lang/rust/pull/{any-id-from-the-list}` to +/// check for more details regarding each change. +/// +/// If you make any major changes (such as adding new values or changing default values), +/// please ensure that the associated PR ID is added to the end of this list. +/// This is necessary because the list must be sorted by the merge date. +pub const CONFIG_CHANGE_HISTORY: &[usize] = &[115898, 116998, 117435, 116881]; /// Extra --check-cfg to add when building /// (Mode restriction, config name, config values (if any)) @@ -130,18 +96,9 @@ const EXTRA_CHECK_CFGS: &[(Option<Mode>, &str, Option<&[&'static str]>)] = &[ (Some(Mode::Std), "freebsd13", None), (Some(Mode::Std), "backtrace_in_libstd", None), /* Extra values not defined in the built-in targets yet, but used in std */ - // #[cfg(bootstrap)] - (Some(Mode::Std), "target_vendor", Some(&["unikraft"])), (Some(Mode::Std), "target_env", Some(&["libnx"])), - // #[cfg(bootstrap)] hurd - (Some(Mode::Std), "target_os", Some(&["teeos", "hurd"])), - (Some(Mode::Rustc), "target_os", Some(&["hurd"])), - // #[cfg(bootstrap)] mips32r6, mips64r6 - ( - Some(Mode::Std), - "target_arch", - Some(&["asmjs", "spirv", "nvptx", "xtensa", "mips32r6", "mips64r6", "csky"]), - ), + // (Some(Mode::Std), "target_os", Some(&[])), + (Some(Mode::Std), "target_arch", Some(&["asmjs", "spirv", "nvptx", "xtensa"])), /* Extra names used by dependencies */ // FIXME: Used by serde_json, but we should not be triggering on external dependencies. (Some(Mode::Rustc), "no_btreemap_remove_entry", None), @@ -211,12 +168,12 @@ pub struct Build { src: PathBuf, out: PathBuf, bootstrap_out: PathBuf, - cargo_info: channel::GitInfo, - rust_analyzer_info: channel::GitInfo, - clippy_info: channel::GitInfo, - miri_info: channel::GitInfo, - rustfmt_info: channel::GitInfo, - in_tree_llvm_info: channel::GitInfo, + cargo_info: GitInfo, + rust_analyzer_info: GitInfo, + clippy_info: GitInfo, + miri_info: GitInfo, + rustfmt_info: GitInfo, + in_tree_llvm_info: GitInfo, local_rebuild: bool, fail_fast: bool, doc_tests: DocTests, @@ -249,7 +206,7 @@ pub struct Build { prerelease_version: Cell<Option<u32>>, #[cfg(feature = "build-metrics")] - metrics: metrics::BuildMetrics, + metrics: crate::utils::metrics::BuildMetrics, } #[derive(Debug, Clone)] @@ -360,6 +317,10 @@ impl Build { // https://github.com/rust-lang/rust/blob/a8a33cf27166d3eabaffc58ed3799e054af3b0c6/src/bootstrap/bootstrap.py#L796-L797 let is_sudo = match env::var_os("SUDO_USER") { Some(_sudo_user) => { + // SAFETY: getuid() system call is always successful and no return value is reserved + // to indicate an error. + // + // For more context, see https://man7.org/linux/man-pages/man2/geteuid.2.html let uid = unsafe { libc::getuid() }; uid == 0 } @@ -369,16 +330,15 @@ impl Build { let is_sudo = false; let omit_git_hash = config.omit_git_hash; - let rust_info = channel::GitInfo::new(omit_git_hash, &src); - let cargo_info = channel::GitInfo::new(omit_git_hash, &src.join("src/tools/cargo")); - let rust_analyzer_info = - channel::GitInfo::new(omit_git_hash, &src.join("src/tools/rust-analyzer")); - let clippy_info = channel::GitInfo::new(omit_git_hash, &src.join("src/tools/clippy")); - let miri_info = channel::GitInfo::new(omit_git_hash, &src.join("src/tools/miri")); - let rustfmt_info = channel::GitInfo::new(omit_git_hash, &src.join("src/tools/rustfmt")); + let rust_info = GitInfo::new(omit_git_hash, &src); + let cargo_info = GitInfo::new(omit_git_hash, &src.join("src/tools/cargo")); + let rust_analyzer_info = GitInfo::new(omit_git_hash, &src.join("src/tools/rust-analyzer")); + let clippy_info = GitInfo::new(omit_git_hash, &src.join("src/tools/clippy")); + let miri_info = GitInfo::new(omit_git_hash, &src.join("src/tools/miri")); + let rustfmt_info = GitInfo::new(omit_git_hash, &src.join("src/tools/rustfmt")); // we always try to use git for LLVM builds - let in_tree_llvm_info = channel::GitInfo::new(false, &src.join("src/llvm-project")); + let in_tree_llvm_info = GitInfo::new(false, &src.join("src/llvm-project")); let initial_target_libdir_str = if config.dry_run() { "/dummy/lib/path/to/lib/".to_string() @@ -471,7 +431,7 @@ impl Build { prerelease_version: Cell::new(None), #[cfg(feature = "build-metrics")] - metrics: metrics::BuildMetrics::init(), + metrics: crate::utils::metrics::BuildMetrics::init(), }; // If local-rust is the same major.minor as the current version, then force a @@ -490,7 +450,7 @@ impl Build { } build.verbose("finding compilers"); - cc_detect::find(&build); + utils::cc_detect::find(&build); // When running `setup`, the profile is about to change, so any requirements we have now may // be different on the next invocation. Don't check for them until the next time x.py is // run. This is ok because `setup` never runs any build commands, so it won't fail if commands are missing. @@ -498,7 +458,7 @@ impl Build { // Similarly, for `setup` we don't actually need submodules or cargo metadata. if !matches!(build.config.cmd, Subcommand::Setup { .. }) { build.verbose("running sanity check"); - sanity::check(&mut build); + crate::core::sanity::check(&mut build); // Make sure we update these before gathering metadata so we don't get an error about missing // Cargo.toml files. @@ -510,7 +470,7 @@ impl Build { build.update_existing_submodules(); build.verbose("learning about cargo"); - metadata::build(&mut build); + crate::core::metadata::build(&mut build); } // Make a symbolic link so we can use a consistent directory in the documentation. @@ -546,7 +506,7 @@ impl Build { // NOTE: The check for the empty directory is here because when running x.py the first time, // the submodule won't be checked out. Check it out now so we can build it. - if !channel::GitInfo::new(false, &absolute_path).is_managed_git_subrepository() + if !GitInfo::new(false, &absolute_path).is_managed_git_subrepository() && !dir_is_empty(&absolute_path) { return; @@ -620,15 +580,19 @@ impl Build { } // Save any local changes, but avoid running `git stash pop` if there are none (since it will exit with an error). - #[allow(deprecated)] // diff-index reports the modifications through the exit status - let has_local_modifications = self - .config - .try_run( + // diff-index reports the modifications through the exit status + let has_local_modifications = !self.run_cmd( + BootstrapCommand::from( Command::new("git") .args(&["diff-index", "--quiet", "HEAD"]) .current_dir(&absolute_path), ) - .is_err(); + .allow_failure() + .output_mode(match self.is_verbose() { + true => OutputMode::PrintAll, + false => OutputMode::PrintOutput, + }), + ); if has_local_modifications { self.run(Command::new("git").args(&["stash", "push"]).current_dir(&absolute_path)); } @@ -660,7 +624,7 @@ impl Build { // Sample output: `submodule.src/rust-installer.path src/tools/rust-installer` let submodule = Path::new(line.splitn(2, ' ').nth(1).unwrap()); // Don't update the submodule unless it's already been cloned. - if channel::GitInfo::new(false, submodule).is_managed_git_subrepository() { + if GitInfo::new(false, submodule).is_managed_git_subrepository() { self.update_submodule(submodule); } } @@ -669,7 +633,7 @@ impl Build { /// Executes the entire build, as configured by the flags and configuration. pub fn build(&mut self) { unsafe { - job::setup(self); + crate::utils::job::setup(self); } // Download rustfmt early so that it can be used in rust-analyzer configs. @@ -678,10 +642,14 @@ impl Build { // hardcoded subcommands match &self.config.cmd { Subcommand::Format { check } => { - return format::format(&builder::Builder::new(&self), *check, &self.config.paths); + return core::build_steps::format::format( + &builder::Builder::new(&self), + *check, + &self.config.paths, + ); } Subcommand::Suggest { run } => { - return suggest::suggest(&builder::Builder::new(&self), *run); + return core::build_steps::suggest::suggest(&builder::Builder::new(&self), *run); } _ => (), } @@ -957,55 +925,103 @@ impl Build { /// Runs a command, printing out nice contextual information if it fails. fn run(&self, cmd: &mut Command) { - if self.config.dry_run() { - return; - } - self.verbose(&format!("running: {cmd:?}")); - run(cmd, self.is_verbose()) + self.run_cmd(BootstrapCommand::from(cmd).fail_fast().output_mode( + match self.is_verbose() { + true => OutputMode::PrintAll, + false => OutputMode::PrintOutput, + }, + )); + } + + /// Runs a command, printing out contextual info if it fails, and delaying errors until the build finishes. + pub(crate) fn run_delaying_failure(&self, cmd: &mut Command) -> bool { + self.run_cmd(BootstrapCommand::from(cmd).delay_failure().output_mode( + match self.is_verbose() { + true => OutputMode::PrintAll, + false => OutputMode::PrintOutput, + }, + )) } /// Runs a command, printing out nice contextual information if it fails. fn run_quiet(&self, cmd: &mut Command) { - if self.config.dry_run() { - return; - } - self.verbose(&format!("running: {cmd:?}")); - run_suppressed(cmd) + self.run_cmd( + BootstrapCommand::from(cmd).fail_fast().output_mode(OutputMode::SuppressOnSuccess), + ); } /// Runs a command, printing out nice contextual information if it fails. /// Exits if the command failed to execute at all, otherwise returns its /// `status.success()`. fn run_quiet_delaying_failure(&self, cmd: &mut Command) -> bool { + self.run_cmd( + BootstrapCommand::from(cmd).delay_failure().output_mode(OutputMode::SuppressOnSuccess), + ) + } + + /// A centralized function for running commands that do not return output. + pub(crate) fn run_cmd<'a, C: Into<BootstrapCommand<'a>>>(&self, cmd: C) -> bool { if self.config.dry_run() { return true; } - if !self.fail_fast { - self.verbose(&format!("running: {cmd:?}")); - if !try_run_suppressed(cmd) { - let mut failures = self.delayed_failures.borrow_mut(); - failures.push(format!("{cmd:?}")); - return false; + + let command = cmd.into(); + self.verbose(&format!("running: {command:?}")); + + let (output, print_error) = match command.output_mode { + mode @ (OutputMode::PrintAll | OutputMode::PrintOutput) => ( + command.command.status().map(|status| Output { + status, + stdout: Vec::new(), + stderr: Vec::new(), + }), + matches!(mode, OutputMode::PrintAll), + ), + OutputMode::SuppressOnSuccess => (command.command.output(), true), + }; + + let output = match output { + Ok(output) => output, + Err(e) => fail(&format!("failed to execute command: {:?}\nerror: {}", command, e)), + }; + let result = if !output.status.success() { + if print_error { + println!( + "\n\ncommand did not execute successfully: {:?}\n\ + expected success, got: {}\n\n\ + stdout ----\n{}\n\ + stderr ----\n{}\n\n", + command.command, + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); } + Err(()) } else { - self.run_quiet(cmd); - } - true - } + Ok(()) + }; - /// Runs a command, printing out contextual info if it fails, and delaying errors until the build finishes. - pub(crate) fn run_delaying_failure(&self, cmd: &mut Command) -> bool { - if !self.fail_fast { - #[allow(deprecated)] // can't use Build::try_run, that's us - if self.config.try_run(cmd).is_err() { - let mut failures = self.delayed_failures.borrow_mut(); - failures.push(format!("{cmd:?}")); - return false; + match result { + Ok(_) => true, + Err(_) => { + match command.failure_behavior { + BehaviorOnFailure::DelayFail => { + if self.fail_fast { + exit!(1); + } + + let mut failures = self.delayed_failures.borrow_mut(); + failures.push(format!("{command:?}")); + } + BehaviorOnFailure::Exit => { + exit!(1); + } + BehaviorOnFailure::Ignore => {} + } + false } - } else { - self.run(cmd); } - true } pub fn is_verbose_than(&self, level: usize) -> bool { @@ -1062,7 +1078,7 @@ impl Build { /// Return a `Group` guard for a [`Step`] that is built for each `--stage`. /// - /// [`Step`]: crate::builder::Step + /// [`Step`]: crate::core::builder::Step #[must_use = "Groups should not be dropped until the Step finishes running"] #[track_caller] fn msg( @@ -1090,7 +1106,7 @@ impl Build { /// Return a `Group` guard for a [`Step`] that is only built once and isn't affected by `--stage`. /// - /// [`Step`]: crate::builder::Step + /// [`Step`]: crate::core::builder::Step #[must_use = "Groups should not be dropped until the Step finishes running"] #[track_caller] fn msg_unstaged( @@ -1182,11 +1198,10 @@ impl Build { .filter(|s| !s.starts_with("-O") && !s.starts_with("/O")) .collect::<Vec<String>>(); - // If we're compiling on macOS then we add a few unconditional flags - // indicating that we want libc++ (more filled out than libstdc++) and - // we want to compile for 10.7. This way we can ensure that + // If we're compiling C++ on macOS then we add a flag indicating that + // we want libc++ (more filled out than libstdc++), ensuring that // LLVM/etc are all properly compiled. - if target.contains("apple-darwin") { + if matches!(c, CLang::Cxx) && target.contains("apple-darwin") { base.push("-stdlib=libc++".into()); } @@ -1250,7 +1265,7 @@ impl Build { // that are only existed in CXX libraries Some(self.cxx.borrow()[&target].path().into()) } else if target != self.config.build - && util::use_host_linker(target) + && helpers::use_host_linker(target) && !target.contains("msvc") { Some(self.cc(target)) @@ -1275,7 +1290,7 @@ impl Build { options[0] = Some("-Clink-arg=-fuse-ld=lld".to_string()); } - let no_threads = util::lld_flag_no_threads(target.contains("windows")); + let no_threads = helpers::lld_flag_no_threads(target.contains("windows")); options[1] = Some(format!("-Clink-arg=-Wl,{no_threads}")); } @@ -1415,7 +1430,7 @@ impl Build { fn extract_beta_rev_from_file<P: AsRef<Path>>(version_file: P) -> Option<String> { let version = fs::read_to_string(version_file).ok()?; - extract_beta_rev(&version) + helpers::extract_beta_rev(&version) } if let Some(s) = self.prerelease_version.get() { @@ -1553,7 +1568,7 @@ impl Build { if !stamp.exists() { eprintln!( - "Error: Unable to find the stamp file {}, did you try to keep a nonexistent build stage?", + "ERROR: Unable to find the stamp file {}, did you try to keep a nonexistent build stage?", stamp.display() ); crate::exit!(1); @@ -1682,7 +1697,7 @@ impl Build { self.verbose_than(1, &format!("Install {src:?} to {dst:?}")); t!(fs::create_dir_all(dstdir)); if !src.exists() { - panic!("Error: File \"{}\" not found!", src.display()); + panic!("ERROR: File \"{}\" not found!", src.display()); } self.copy_internal(src, &dst, true); chmod(&dst, perms); @@ -1729,7 +1744,7 @@ impl Build { /// Returns if config.ninja is enabled, and checks for ninja existence, /// exiting with a nicer error message if not. fn ninja(&self) -> bool { - let mut cmd_finder = crate::sanity::Finder::new(); + let mut cmd_finder = crate::core::sanity::Finder::new(); if self.config.ninja_in_file { // Some Linux distros rename `ninja` to `ninja-build`. @@ -1795,17 +1810,6 @@ to download LLVM rather than building it. } } -/// Extract the beta revision from the full version string. -/// -/// The full version string looks like "a.b.c-beta.y". And we need to extract -/// the "y" part from the string. -pub fn extract_beta_rev(version: &str) -> Option<String> { - let parts = version.splitn(2, "-beta.").collect::<Vec<_>>(); - let count = parts.get(1).and_then(|s| s.find(' ').map(|p| (&s[..p]).to_string())); - - count -} - #[cfg(unix)] fn chmod(path: &Path, perms: u32) { use std::os::unix::fs::*; @@ -1844,3 +1848,27 @@ fn envify(s: &str) -> String { .flat_map(|c| c.to_uppercase()) .collect() } + +pub fn find_recent_config_change_ids(current_id: usize) -> Vec<usize> { + if !CONFIG_CHANGE_HISTORY.contains(¤t_id) { + // If the current change-id is greater than the most recent one, return + // an empty list (it may be due to switching from a recent branch to an + // older one); otherwise, return the full list (assuming the user provided + // the incorrect change-id by accident). + if let Some(max_id) = CONFIG_CHANGE_HISTORY.iter().max() { + if ¤t_id > max_id { + return Vec::new(); + } + } + + return CONFIG_CHANGE_HISTORY.to_vec(); + } + + let index = CONFIG_CHANGE_HISTORY.iter().position(|&id| id == current_id).unwrap(); + + CONFIG_CHANGE_HISTORY + .iter() + .skip(index + 1) // Skip the current_id and IDs before it + .cloned() + .collect() +} diff --git a/src/bootstrap/builder/tests.rs b/src/bootstrap/src/tests/builder.rs index 80e66622e..96139f7b0 100644 --- a/src/bootstrap/builder/tests.rs +++ b/src/bootstrap/src/tests/builder.rs @@ -1,6 +1,6 @@ use super::*; -use crate::config::{Config, DryRun, TargetSelection}; -use crate::doc::DocumentationFormat; +use crate::core::config::{Config, DryRun, TargetSelection}; +use crate::core::build_steps::doc::DocumentationFormat; use std::thread; fn configure(cmd: &str, host: &[&str], target: &[&str]) -> Config { @@ -22,7 +22,6 @@ fn configure_with_args(cmd: &[String], host: &[&str], target: &[&str]) -> Config ..Config::parse(&["check".to_owned()]) }); submodule_build.update_submodule(Path::new("src/doc/book")); - submodule_build.update_submodule(Path::new("src/tools/rust-analyzer")); config.submodules = Some(false); config.ninja_in_file = false; @@ -159,7 +158,7 @@ fn alias_and_path_for_library() { #[test] fn test_beta_rev_parsing() { - use crate::extract_beta_rev; + use crate::utils::helpers::extract_beta_rev; // single digit revision assert_eq!(extract_beta_rev("1.99.9-beta.7 (xxxxxx)"), Some("7".to_string())); @@ -175,7 +174,7 @@ fn test_beta_rev_parsing() { mod defaults { use super::{configure, first, run_build}; - use crate::builder::*; + use crate::core::builder::*; use crate::Config; use pretty_assertions::assert_eq; @@ -286,7 +285,7 @@ mod defaults { mod dist { use super::{first, run_build, Config}; - use crate::builder::*; + use crate::core::builder::*; use pretty_assertions::assert_eq; fn configure(host: &[&str], target: &[&str]) -> Config { diff --git a/src/bootstrap/config/tests.rs b/src/bootstrap/src/tests/config.rs index aac76cdcb..59bd52a94 100644 --- a/src/bootstrap/config/tests.rs +++ b/src/bootstrap/src/tests/config.rs @@ -1,9 +1,14 @@ -use crate::config::TomlConfig; - +use crate::core::config::TomlConfig; use super::{Config, Flags}; + use clap::CommandFactory; use serde::Deserialize; -use std::{env, path::Path}; +use std::{ + env, + fs::{remove_file, File}, + io::Write, + path::Path, +}; fn parse(config: &str) -> Config { Config::parse_inner(&["check".to_owned(), "--config=/does/not/exist".to_owned()], |&_| { @@ -13,7 +18,7 @@ fn parse(config: &str) -> Config { #[test] fn download_ci_llvm() { - if crate::llvm::is_ci_llvm_modified(&parse("")) { + if crate::core::build_steps::llvm::is_ci_llvm_modified(&parse("")) { eprintln!("Detected LLVM as non-available: running in CI and modified LLVM in this change"); return; } @@ -102,7 +107,7 @@ fn override_toml() { &[ "check".to_owned(), "--config=/does/not/exist".to_owned(), - "--set=changelog-seen=1".to_owned(), + "--set=change-id=1".to_owned(), "--set=rust.lto=fat".to_owned(), "--set=rust.deny-warnings=false".to_owned(), "--set=build.gdb=\"bar\"".to_owned(), @@ -112,7 +117,7 @@ fn override_toml() { |&_| { toml::from_str( r#" -changelog-seen = 0 +change-id = 0 [rust] lto = "off" deny-warnings = true @@ -129,10 +134,10 @@ build-config = {} .unwrap() }, ); - assert_eq!(config.changelog_seen, Some(1), "setting top-level value"); + assert_eq!(config.change_id, Some(1), "setting top-level value"); assert_eq!( config.rust_lto, - crate::config::RustcLto::Fat, + crate::core::config::RustcLto::Fat, "setting string value without quotes" ); assert_eq!(config.gdb, Some("bar".into()), "setting string value with quotes"); @@ -156,10 +161,10 @@ fn override_toml_duplicate() { &[ "check".to_owned(), "--config=/does/not/exist".to_owned(), - "--set=changelog-seen=1".to_owned(), - "--set=changelog-seen=2".to_owned(), + "--set=change-id=1".to_owned(), + "--set=change-id=2".to_owned(), ], - |&_| toml::from_str("changelog-seen = 0").unwrap(), + |&_| toml::from_str("change-id = 0").unwrap(), ); } @@ -170,7 +175,7 @@ fn profile_user_dist() { "profile = \"user\"".to_owned() } else { assert!(file.ends_with("config.dist.toml")); - std::fs::read_to_string(dbg!(file)).unwrap() + std::fs::read_to_string(file).unwrap() }; toml::from_str(&contents) .and_then(|table: toml::Value| TomlConfig::deserialize(table)) @@ -196,3 +201,19 @@ fn rust_optimize() { fn invalid_rust_optimize() { parse("rust.optimize = \"a\""); } + +#[test] +fn verify_file_integrity() { + let config = parse(""); + + let tempfile = config.tempdir().join(".tmp-test-file"); + File::create(&tempfile).unwrap().write_all(b"dummy value").unwrap(); + assert!(tempfile.exists()); + + assert!( + config + .verify(&tempfile, "7e255dd9542648a8779268a0f268b891a198e9828e860ed23f826440e786eae5") + ); + + remove_file(tempfile).unwrap(); +} diff --git a/src/bootstrap/setup/tests.rs b/src/bootstrap/src/tests/setup.rs index 0fe6e4a46..0fe6e4a46 100644 --- a/src/bootstrap/setup/tests.rs +++ b/src/bootstrap/src/tests/setup.rs diff --git a/src/bootstrap/bin/_helper.rs b/src/bootstrap/src/utils/bin_helpers.rs index 09aa471db..c90fd2805 100644 --- a/src/bootstrap/bin/_helper.rs +++ b/src/bootstrap/src/utils/bin_helpers.rs @@ -1,8 +1,12 @@ +//! This file is meant to be included directly from bootstrap shims to avoid a +//! dependency on the bootstrap library. This reduces the binary size and +//! improves compilation time by reducing the linking time. + /// Parses the value of the "RUSTC_VERBOSE" environment variable and returns it as a `usize`. /// If it was not defined, returns 0 by default. /// /// Panics if "RUSTC_VERBOSE" is defined with the value that is not an unsigned integer. -fn parse_rustc_verbose() -> usize { +pub(crate) fn parse_rustc_verbose() -> usize { use std::str::FromStr; match std::env::var("RUSTC_VERBOSE") { @@ -14,11 +18,11 @@ fn parse_rustc_verbose() -> usize { /// Parses the value of the "RUSTC_STAGE" environment variable and returns it as a `String`. /// /// If "RUSTC_STAGE" was not set, the program will be terminated with 101. -fn parse_rustc_stage() -> String { +pub(crate) fn parse_rustc_stage() -> String { std::env::var("RUSTC_STAGE").unwrap_or_else(|_| { // Don't panic here; it's reasonable to try and run these shims directly. Give a helpful error instead. - eprintln!("rustc shim: fatal: RUSTC_STAGE was not set"); - eprintln!("rustc shim: note: use `x.py build -vvv` to see all environment variables set by bootstrap"); - exit(101); + eprintln!("rustc shim: FATAL: RUSTC_STAGE was not set"); + eprintln!("rustc shim: NOTE: use `x.py build -vvv` to see all environment variables set by bootstrap"); + std::process::exit(101); }) } diff --git a/src/bootstrap/cache.rs b/src/bootstrap/src/utils/cache.rs index 53e4ff034..1b2aa9c23 100644 --- a/src/bootstrap/cache.rs +++ b/src/bootstrap/src/utils/cache.rs @@ -14,7 +14,7 @@ use std::sync::Mutex; // FIXME: replace with std::lazy after it gets stabilized and reaches beta use once_cell::sync::Lazy; -use crate::builder::Step; +use crate::core::builder::Step; pub struct Interned<T>(usize, PhantomData<*const T>); diff --git a/src/bootstrap/cc_detect.rs b/src/bootstrap/src/utils/cc_detect.rs index 2496c2a9d..52b36ce75 100644 --- a/src/bootstrap/cc_detect.rs +++ b/src/bootstrap/src/utils/cc_detect.rs @@ -26,8 +26,8 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::{env, iter}; -use crate::config::{Target, TargetSelection}; -use crate::util::output; +use crate::core::config::TargetSelection; +use crate::utils::helpers::output; use crate::{Build, CLang, GitRepo}; // The `cc` crate doesn't provide a way to obtain a path to the detected archiver, @@ -107,10 +107,11 @@ pub fn find(build: &Build) { pub fn find_target(build: &Build, target: TargetSelection) { let mut cfg = new_cc_build(build, target); let config = build.config.target_config.get(&target); - if let Some(cc) = config.and_then(|c| c.cc.as_ref()) { + if let Some(cc) = config + .and_then(|c| c.cc.clone()) + .or_else(|| default_compiler(&mut cfg, Language::C, target, build)) + { cfg.compiler(cc); - } else { - set_compiler(&mut cfg, Language::C, target, config, build); } let compiler = cfg.get_compiler(); @@ -127,12 +128,12 @@ pub fn find_target(build: &Build, target: TargetSelection) { // We'll need one anyways if the target triple is also a host triple let mut cfg = new_cc_build(build, target); cfg.cpp(true); - let cxx_configured = if let Some(cxx) = config.and_then(|c| c.cxx.as_ref()) { + let cxx_configured = if let Some(cxx) = config + .and_then(|c| c.cxx.clone()) + .or_else(|| default_compiler(&mut cfg, Language::CPlusPlus, target, build)) + { cfg.compiler(cxx); true - } else if build.hosts.contains(&target) || build.build == target { - set_compiler(&mut cfg, Language::CPlusPlus, target, config, build); - true } else { // Use an auto-detected compiler (or one configured via `CXX_target_triple` env vars). cfg.try_get_compiler().is_ok() @@ -161,22 +162,21 @@ pub fn find_target(build: &Build, target: TargetSelection) { } } -fn set_compiler( +fn default_compiler( cfg: &mut cc::Build, compiler: Language, target: TargetSelection, - config: Option<&Target>, build: &Build, -) { +) -> Option<PathBuf> { match &*target.triple { // When compiling for android we may have the NDK configured in the // config.toml in which case we look there. Otherwise the default // compiler already takes into account the triple in question. - t if t.contains("android") => { - if let Some(ndk) = config.and_then(|c| c.ndk.as_ref()) { - cfg.compiler(ndk_compiler(compiler, &*target.triple, ndk)); - } - } + t if t.contains("android") => build + .config + .android_ndk + .as_ref() + .map(|ndk| ndk_compiler(compiler, &*target.triple, ndk)), // The default gcc version from OpenBSD may be too old, try using egcc, // which is a gcc version from ports, if this is the case. @@ -184,45 +184,48 @@ fn set_compiler( let c = cfg.get_compiler(); let gnu_compiler = compiler.gcc(); if !c.path().ends_with(gnu_compiler) { - return; + return None; } let output = output(c.to_command().arg("--version")); - let i = match output.find(" 4.") { - Some(i) => i, - None => return, - }; + let i = output.find(" 4.")?; match output[i + 3..].chars().next().unwrap() { '0'..='6' => {} - _ => return, + _ => return None, } let alternative = format!("e{gnu_compiler}"); if Command::new(&alternative).output().is_ok() { - cfg.compiler(alternative); + Some(PathBuf::from(alternative)) + } else { + None } } - "mips-unknown-linux-musl" => { + "mips-unknown-linux-musl" if compiler == Language::C => { if cfg.get_compiler().path().to_str() == Some("gcc") { - cfg.compiler("mips-linux-musl-gcc"); + Some(PathBuf::from("mips-linux-musl-gcc")) + } else { + None } } - "mipsel-unknown-linux-musl" => { + "mipsel-unknown-linux-musl" if compiler == Language::C => { if cfg.get_compiler().path().to_str() == Some("gcc") { - cfg.compiler("mipsel-linux-musl-gcc"); + Some(PathBuf::from("mipsel-linux-musl-gcc")) + } else { + None } } - t if t.contains("musl") => { + t if t.contains("musl") && compiler == Language::C => { if let Some(root) = build.musl_root(target) { let guess = root.join("bin/musl-gcc"); - if guess.exists() { - cfg.compiler(guess); - } + if guess.exists() { Some(guess) } else { None } + } else { + None } } - _ => {} + _ => None, } } @@ -243,10 +246,22 @@ pub(crate) fn ndk_compiler(compiler: Language, triple: &str, ndk: &Path) -> Path let api_level = if triple.contains("aarch64") || triple.contains("x86_64") { "21" } else { "19" }; let compiler = format!("{}{}-{}", triple_translated, api_level, compiler.clang()); - ndk.join("bin").join(compiler) + let host_tag = if cfg!(target_os = "macos") { + // The NDK uses universal binaries, so this is correct even on ARM. + "darwin-x86_64" + } else if cfg!(target_os = "windows") { + "windows-x86_64" + } else { + // NDK r25b only has official releases for macOS, Windows and Linux. + // Try the Linux directory everywhere else, on the assumption that the OS has an + // emulation layer that can cope (e.g. BSDs). + "linux-x86_64" + }; + ndk.join("toolchains").join("llvm").join("prebuilt").join(host_tag).join("bin").join(compiler) } /// The target programming language for a native compiler. +#[derive(PartialEq)] pub(crate) enum Language { /// The compiler is targeting C. C, diff --git a/src/bootstrap/channel.rs b/src/bootstrap/src/utils/channel.rs index 870185740..e59d7f22a 100644 --- a/src/bootstrap/channel.rs +++ b/src/bootstrap/src/utils/channel.rs @@ -9,8 +9,7 @@ use std::fs; use std::path::Path; use std::process::Command; -use crate::util::output; -use crate::util::t; +use crate::utils::helpers::{output, t}; use crate::Build; #[derive(Clone, Default)] diff --git a/src/bootstrap/dylib_util.rs b/src/bootstrap/src/utils/dylib.rs index b14c0bed6..279a6a010 100644 --- a/src/bootstrap/dylib_util.rs +++ b/src/bootstrap/src/utils/dylib.rs @@ -1,7 +1,4 @@ -// Various utilities for working with dylib paths. -// -// This file is meant to be included directly to avoid a dependency on the bootstrap library from -// the rustc and rustdoc wrappers. This improves compilation time by reducing the linking time. +//! Various utilities for working with dylib paths. /// Returns the environment variable which the dynamic library lookup path /// resides in for this platform. @@ -21,10 +18,10 @@ pub fn dylib_path_var() -> &'static str { /// Parses the `dylib_path_var()` environment variable, returning a list of /// paths that are members of this lookup path. -pub fn dylib_path() -> Vec<PathBuf> { - let var = match env::var_os(dylib_path_var()) { +pub fn dylib_path() -> Vec<std::path::PathBuf> { + let var = match std::env::var_os(dylib_path_var()) { Some(v) => v, None => return vec![], }; - env::split_paths(&var).collect() + std::env::split_paths(&var).collect() } diff --git a/src/bootstrap/src/utils/exec.rs b/src/bootstrap/src/utils/exec.rs new file mode 100644 index 000000000..0aede2022 --- /dev/null +++ b/src/bootstrap/src/utils/exec.rs @@ -0,0 +1,60 @@ +use std::process::Command; + +/// What should be done when the command fails. +#[derive(Debug, Copy, Clone)] +pub enum BehaviorOnFailure { + /// Immediately stop bootstrap. + Exit, + /// Delay failure until the end of bootstrap invocation. + DelayFail, + /// Ignore the failure, the command can fail in an expected way. + Ignore, +} + +/// How should the output of the command be handled. +#[derive(Debug, Copy, Clone)] +pub enum OutputMode { + /// Print both the output (by inheriting stdout/stderr) and also the command itself, if it + /// fails. + PrintAll, + /// Print the output (by inheriting stdout/stderr). + PrintOutput, + /// Suppress the output if the command succeeds, otherwise print the output. + SuppressOnSuccess, +} + +/// Wrapper around `std::process::Command`. +#[derive(Debug)] +pub struct BootstrapCommand<'a> { + pub command: &'a mut Command, + pub failure_behavior: BehaviorOnFailure, + pub output_mode: OutputMode, +} + +impl<'a> BootstrapCommand<'a> { + pub fn delay_failure(self) -> Self { + Self { failure_behavior: BehaviorOnFailure::DelayFail, ..self } + } + + pub fn fail_fast(self) -> Self { + Self { failure_behavior: BehaviorOnFailure::Exit, ..self } + } + + pub fn allow_failure(self) -> Self { + Self { failure_behavior: BehaviorOnFailure::Ignore, ..self } + } + + pub fn output_mode(self, output_mode: OutputMode) -> Self { + Self { output_mode, ..self } + } +} + +impl<'a> From<&'a mut Command> for BootstrapCommand<'a> { + fn from(command: &'a mut Command) -> Self { + Self { + command, + failure_behavior: BehaviorOnFailure::Exit, + output_mode: OutputMode::PrintAll, + } + } +} diff --git a/src/bootstrap/util.rs b/src/bootstrap/src/utils/helpers.rs index 3c4a21434..5bc81f2d9 100644 --- a/src/bootstrap/util.rs +++ b/src/bootstrap/src/utils/helpers.rs @@ -3,7 +3,7 @@ //! Simple things like testing the various filesystem operations here and there, //! not a lot of interesting happenings here unfortunately. -use build_helper::util::{fail, try_run}; +use build_helper::util::fail; use std::env; use std::fs; use std::io; @@ -12,10 +12,12 @@ use std::process::{Command, Stdio}; use std::str; use std::time::{Instant, SystemTime, UNIX_EPOCH}; -use crate::builder::Builder; -use crate::config::{Config, TargetSelection}; +use crate::core::builder::Builder; +use crate::core::config::{Config, TargetSelection}; use crate::OnceCell; +pub use crate::utils::dylib::{dylib_path, dylib_path_var}; + /// A helper macro to `unwrap` a result except also print out details like: /// /// * The file/line of the panic @@ -81,8 +83,6 @@ pub fn add_dylib_path(path: Vec<PathBuf>, cmd: &mut Command) { cmd.env(dylib_path_var(), t!(env::join_paths(list))); } -include!("dylib_util.rs"); - /// Adds a list of lookup paths to `cmd`'s link library lookup path. pub fn add_link_lib_path(path: Vec<PathBuf>, cmd: &mut Command) { let mut list = link_lib_path(); @@ -183,6 +183,19 @@ pub fn use_host_linker(target: TargetSelection) -> bool { || target.contains("switch")) } +pub fn target_supports_cranelift_backend(target: TargetSelection) -> bool { + if target.contains("linux") { + target.contains("x86_64") + || target.contains("aarch64") + || target.contains("s390x") + || target.contains("riscv64gc") + } else if target.contains("darwin") || target.contains("windows") { + target.contains("x86_64") + } else { + false + } +} + pub fn is_valid_test_suite_arg<'a, P: AsRef<Path>>( path: &'a Path, suite_path: P, @@ -216,17 +229,11 @@ pub fn is_valid_test_suite_arg<'a, P: AsRef<Path>>( } } -pub fn run(cmd: &mut Command, print_cmd_on_fail: bool) { - if try_run(cmd, print_cmd_on_fail).is_err() { - crate::exit!(1); - } -} - pub fn check_run(cmd: &mut Command, print_cmd_on_fail: bool) -> bool { let status = match cmd.status() { Ok(status) => status, Err(e) => { - println!("failed to execute command: {cmd:?}\nerror: {e}"); + println!("failed to execute command: {cmd:?}\nERROR: {e}"); return false; } }; @@ -239,32 +246,6 @@ pub fn check_run(cmd: &mut Command, print_cmd_on_fail: bool) -> bool { status.success() } -pub fn run_suppressed(cmd: &mut Command) { - if !try_run_suppressed(cmd) { - crate::exit!(1); - } -} - -pub fn try_run_suppressed(cmd: &mut Command) -> bool { - let output = match cmd.output() { - Ok(status) => status, - Err(e) => fail(&format!("failed to execute command: {cmd:?}\nerror: {e}")), - }; - if !output.status.success() { - println!( - "\n\ncommand did not execute successfully: {:?}\n\ - expected success, got: {}\n\n\ - stdout ----\n{}\n\ - stderr ----\n{}\n\n", - cmd, - output.status, - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - } - output.status.success() -} - pub fn make(host: &str) -> PathBuf { if host.contains("dragonfly") || host.contains("freebsd") @@ -281,7 +262,7 @@ pub fn make(host: &str) -> PathBuf { pub fn output(cmd: &mut Command) -> String { let output = match cmd.stderr(Stdio::inherit()).output() { Ok(status) => status, - Err(e) => fail(&format!("failed to execute command: {cmd:?}\nerror: {e}")), + Err(e) => fail(&format!("failed to execute command: {cmd:?}\nERROR: {e}")), }; if !output.status.success() { panic!( @@ -293,23 +274,6 @@ pub fn output(cmd: &mut Command) -> String { String::from_utf8(output.stdout).unwrap() } -pub fn output_result(cmd: &mut Command) -> Result<String, String> { - let output = match cmd.stderr(Stdio::inherit()).output() { - Ok(status) => status, - Err(e) => return Err(format!("failed to run command: {cmd:?}: {e}")), - }; - if !output.status.success() { - return Err(format!( - "command did not execute successfully: {:?}\n\ - expected success, got: {}\n{}", - cmd, - output.status, - String::from_utf8(output.stderr).map_err(|err| format!("{err:?}"))? - )); - } - Ok(String::from_utf8(output.stdout).map_err(|err| format!("{err:?}"))?) -} - /// Returns the last-modified time for `path`, or zero if it doesn't exist. pub fn mtime(path: &Path) -> SystemTime { fs::metadata(path).and_then(|f| f.modified()).unwrap_or(UNIX_EPOCH) @@ -363,7 +327,7 @@ pub(crate) fn absolute(path: &Path) -> PathBuf { } #[cfg(not(any(unix, windows)))] { - println!("warning: bootstrap is not supported on non-unix platforms"); + println!("WARNING: bootstrap is not supported on non-unix platforms"); t!(std::fs::canonicalize(t!(std::env::current_dir()))).join(path) } } @@ -495,3 +459,14 @@ pub fn lld_flag_no_threads(is_windows: bool) -> &'static str { pub fn dir_is_empty(dir: &Path) -> bool { t!(std::fs::read_dir(dir)).next().is_none() } + +/// Extract the beta revision from the full version string. +/// +/// The full version string looks like "a.b.c-beta.y". And we need to extract +/// the "y" part from the string. +pub fn extract_beta_rev(version: &str) -> Option<String> { + let parts = version.splitn(2, "-beta.").collect::<Vec<_>>(); + let count = parts.get(1).and_then(|s| s.find(' ').map(|p| (&s[..p]).to_string())); + + count +} diff --git a/src/bootstrap/src/utils/job.rs b/src/bootstrap/src/utils/job.rs new file mode 100644 index 000000000..c5c718a89 --- /dev/null +++ b/src/bootstrap/src/utils/job.rs @@ -0,0 +1,159 @@ +#[cfg(windows)] +pub use for_windows::*; + +#[cfg(any(target_os = "haiku", target_os = "hermit", not(any(unix, windows))))] +pub unsafe fn setup(_build: &mut crate::Build) {} + +#[cfg(all(unix, not(target_os = "haiku")))] +pub unsafe fn setup(build: &mut crate::Build) { + if build.config.low_priority { + libc::setpriority(libc::PRIO_PGRP as _, 0, 10); + } +} + +#[cfg(windows)] +mod for_windows { + //! Job management on Windows for bootstrapping + //! + //! Most of the time when you're running a build system (e.g., make) you expect + //! Ctrl-C or abnormal termination to actually terminate the entire tree of + //! process in play, not just the one at the top. This currently works "by + //! default" on Unix platforms because Ctrl-C actually sends a signal to the + //! *process group* rather than the parent process, so everything will get torn + //! down. On Windows, however, this does not happen and Ctrl-C just kills the + //! parent process. + //! + //! To achieve the same semantics on Windows we use Job Objects to ensure that + //! all processes die at the same time. Job objects have a mode of operation + //! where when all handles to the object are closed it causes all child + //! processes associated with the object to be terminated immediately. + //! Conveniently whenever a process in the job object spawns a new process the + //! child will be associated with the job object as well. This means if we add + //! ourselves to the job object we create then everything will get torn down! + //! + //! Unfortunately most of the time the build system is actually called from a + //! python wrapper (which manages things like building the build system) so this + //! all doesn't quite cut it so far. To go the last mile we duplicate the job + //! object handle into our parent process (a python process probably) and then + //! close our own handle. This means that the only handle to the job object + //! resides in the parent python process, so when python dies the whole build + //! system dies (as one would probably expect!). + //! + //! Note that this module has a #[cfg(windows)] above it as none of this logic + //! is required on Unix. + + use crate::Build; + use std::env; + use std::ffi::c_void; + use std::io; + use std::mem; + + use windows::{ + core::PCWSTR, + Win32::Foundation::{CloseHandle, DuplicateHandle, DUPLICATE_SAME_ACCESS, HANDLE}, + Win32::System::Diagnostics::Debug::{ + SetErrorMode, SEM_NOGPFAULTERRORBOX, THREAD_ERROR_MODE, + }, + Win32::System::JobObjects::{ + AssignProcessToJobObject, CreateJobObjectW, JobObjectExtendedLimitInformation, + SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, + JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOB_OBJECT_LIMIT_PRIORITY_CLASS, + }, + Win32::System::Threading::{ + GetCurrentProcess, OpenProcess, BELOW_NORMAL_PRIORITY_CLASS, PROCESS_DUP_HANDLE, + }, + }; + + pub unsafe fn setup(build: &mut Build) { + // Enable the Windows Error Reporting dialog which msys disables, + // so we can JIT debug rustc + let mode = SetErrorMode(THREAD_ERROR_MODE::default()); + let mode = THREAD_ERROR_MODE(mode); + SetErrorMode(mode & !SEM_NOGPFAULTERRORBOX); + + // Create a new job object for us to use + let job = CreateJobObjectW(None, PCWSTR::null()).unwrap(); + + // Indicate that when all handles to the job object are gone that all + // process in the object should be killed. Note that this includes our + // entire process tree by default because we've added ourselves and our + // children will reside in the job by default. + let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default(); + info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + if build.config.low_priority { + info.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_PRIORITY_CLASS; + info.BasicLimitInformation.PriorityClass = BELOW_NORMAL_PRIORITY_CLASS.0; + } + let r = SetInformationJobObject( + job, + JobObjectExtendedLimitInformation, + &info as *const _ as *const c_void, + mem::size_of_val(&info) as u32, + ); + assert!(r.is_ok(), "{}", io::Error::last_os_error()); + + // Assign our process to this job object. Note that if this fails, one very + // likely reason is that we are ourselves already in a job object! This can + // happen on the build bots that we've got for Windows, or if just anyone + // else is instrumenting the build. In this case we just bail out + // immediately and assume that they take care of it. + // + // Also note that nested jobs (why this might fail) are supported in recent + // versions of Windows, but the version of Windows that our bots are running + // at least don't support nested job objects. + let r = AssignProcessToJobObject(job, GetCurrentProcess()); + if r.is_err() { + CloseHandle(job).ok(); + return; + } + + // If we've got a parent process (e.g., the python script that called us) + // then move ownership of this job object up to them. That way if the python + // script is killed (e.g., via ctrl-c) then we'll all be torn down. + // + // If we don't have a parent (e.g., this was run directly) then we + // intentionally leak the job object handle. When our process exits + // (normally or abnormally) it will close the handle implicitly, causing all + // processes in the job to be cleaned up. + let pid = match env::var("BOOTSTRAP_PARENT_ID") { + Ok(s) => s, + Err(..) => return, + }; + + let parent = match OpenProcess(PROCESS_DUP_HANDLE, false, pid.parse().unwrap()).ok() { + Some(parent) => parent, + _ => { + // If we get a null parent pointer here, it is possible that either + // we have an invalid pid or the parent process has been closed. + // Since the first case rarely happens + // (only when wrongly setting the environmental variable), + // it might be better to improve the experience of the second case + // when users have interrupted the parent process and we haven't finish + // duplicating the handle yet. We just need close the job object if that occurs. + CloseHandle(job).ok(); + return; + } + }; + + let mut parent_handle = HANDLE::default(); + let r = DuplicateHandle( + GetCurrentProcess(), + job, + parent, + &mut parent_handle, + 0, + false, + DUPLICATE_SAME_ACCESS, + ); + + // If this failed, well at least we tried! An example of DuplicateHandle + // failing in the past has been when the wrong python2 package spawned this + // build system (e.g., the `python2` package in MSYS instead of + // `mingw-w64-x86_64-python2`). Not sure why it failed, but the "failure + // mode" here is that we only clean everything up when the build system + // dies, not when the python parent does, so not too bad. + if r.is_err() { + CloseHandle(job).ok(); + } + } +} diff --git a/src/bootstrap/metrics.rs b/src/bootstrap/src/utils/metrics.rs index cf8d33dfc..174f37422 100644 --- a/src/bootstrap/metrics.rs +++ b/src/bootstrap/src/utils/metrics.rs @@ -4,8 +4,8 @@ //! As this module requires additional dependencies not present during local builds, it's cfg'd //! away whenever the `build.metrics` config option is not set to `true`. -use crate::builder::{Builder, Step}; -use crate::util::t; +use crate::core::builder::{Builder, Step}; +use crate::utils::helpers::t; use crate::Build; use build_helper::metrics::{ JsonInvocation, JsonInvocationSystemStats, JsonNode, JsonRoot, JsonStepSystemStats, Test, @@ -180,7 +180,7 @@ impl BuildMetrics { t!(serde_json::from_slice::<JsonRoot>(&contents)).invocations } else { println!( - "warning: overriding existing build/metrics.json, as it's not \ + "WARNING: overriding existing build/metrics.json, as it's not \ compatible with build metrics format version {CURRENT_FORMAT_VERSION}." ); Vec::new() diff --git a/src/bootstrap/src/utils/mod.rs b/src/bootstrap/src/utils/mod.rs new file mode 100644 index 000000000..8ca22d008 --- /dev/null +++ b/src/bootstrap/src/utils/mod.rs @@ -0,0 +1,15 @@ +//! This module contains integral components of the build and configuration process, providing +//! support for a wide range of tasks and operations such as caching, tarballs, release +//! channels, job management, etc. + +pub(crate) mod cache; +pub(crate) mod cc_detect; +pub(crate) mod channel; +pub(crate) mod dylib; +pub(crate) mod exec; +pub(crate) mod helpers; +pub(crate) mod job; +#[cfg(feature = "build-metrics")] +pub(crate) mod metrics; +pub(crate) mod render_tests; +pub(crate) mod tarball; diff --git a/src/bootstrap/render_tests.rs b/src/bootstrap/src/utils/render_tests.rs index 6802bf451..bff47f65c 100644 --- a/src/bootstrap/render_tests.rs +++ b/src/bootstrap/src/utils/render_tests.rs @@ -6,7 +6,7 @@ //! and rustc) libtest doesn't include the rendered human-readable output as a JSON field. We had //! to reimplement all the rendering logic in this module because of that. -use crate::builder::Builder; +use crate::core::builder::Builder; use std::io::{BufRead, BufReader, Read, Write}; use std::process::{ChildStdout, Command, Stdio}; use std::time::Duration; @@ -197,7 +197,7 @@ impl<'a> Renderer<'a> { println!("{stdout}"); } if let Some(message) = &failure.message { - println!("note: {message}"); + println!("NOTE: {message}"); } } } diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/src/utils/tarball.rs index 95d909c57..a8393f88f 100644 --- a/src/bootstrap/tarball.rs +++ b/src/bootstrap/src/utils/tarball.rs @@ -3,9 +3,10 @@ use std::{ process::Command, }; -use crate::builder::Builder; -use crate::channel; -use crate::util::t; +use crate::core::build_steps::dist::distdir; +use crate::core::builder::Builder; +use crate::utils::channel; +use crate::utils::helpers::t; #[derive(Copy, Clone)] pub(crate) enum OverlayKind { @@ -18,6 +19,7 @@ pub(crate) enum OverlayKind { RustDemangler, RLS, RustAnalyzer, + RustcCodegenCranelift, } impl OverlayKind { @@ -57,6 +59,11 @@ impl OverlayKind { "src/tools/rust-analyzer/LICENSE-APACHE", "src/tools/rust-analyzer/LICENSE-MIT", ], + OverlayKind::RustcCodegenCranelift => &[ + "compiler/rustc_codegen_cranelift/Readme.md", + "compiler/rustc_codegen_cranelift/LICENSE-APACHE", + "compiler/rustc_codegen_cranelift/LICENSE-MIT", + ], } } @@ -79,6 +86,7 @@ impl OverlayKind { OverlayKind::RustAnalyzer => builder .rust_analyzer_info .version(builder, &builder.release_num("rust-analyzer/crates/rust-analyzer")), + OverlayKind::RustcCodegenCranelift => builder.rust_version(), } } } @@ -112,7 +120,7 @@ impl<'a> Tarball<'a> { } fn new_inner(builder: &'a Builder<'a>, component: &str, target: Option<String>) -> Self { - let pkgname = crate::dist::pkgname(builder, component); + let pkgname = crate::core::build_steps::dist::pkgname(builder, component); let mut temp_dir = builder.out.join("tmp").join("tarball").join(component); if let Some(target) = &target { @@ -265,7 +273,7 @@ impl<'a> Tarball<'a> { t!(std::fs::rename(&self.image_dir, &dest)); self.run(|this, cmd| { - let distdir = crate::dist::distdir(this.builder); + let distdir = distdir(this.builder); t!(std::fs::create_dir_all(&distdir)); cmd.arg("tarball") .arg("--input") @@ -292,7 +300,7 @@ impl<'a> Tarball<'a> { .arg("--non-installed-overlay") .arg(&self.overlay_dir) .arg("--output-dir") - .arg(crate::dist::distdir(self.builder)); + .arg(distdir(self.builder)); } fn run(self, build_cli: impl FnOnce(&Tarball<'a>, &mut Command)) -> GeneratedTarball { @@ -306,11 +314,11 @@ impl<'a> Tarball<'a> { self.builder.install(&self.builder.src.join(file), &self.overlay_dir, 0o644); } - let mut cmd = self.builder.tool_cmd(crate::tool::Tool::RustInstaller); + let mut cmd = self.builder.tool_cmd(crate::core::build_steps::tool::Tool::RustInstaller); let package_name = self.package_name(); self.builder.info(&format!("Dist {package_name}")); - let _time = crate::util::timeit(self.builder); + let _time = crate::utils::helpers::timeit(self.builder); build_cli(&self, &mut cmd); cmd.arg("--work-dir").arg(&self.temp_dir); @@ -344,7 +352,7 @@ impl<'a> Tarball<'a> { .unwrap_or("gz"); GeneratedTarball { - path: crate::dist::distdir(self.builder).join(format!("{package_name}.tar.{ext}")), + path: distdir(self.builder).join(format!("{package_name}.tar.{ext}")), decompressed_output, work: self.temp_dir, } |