summaryrefslogtreecommitdiffstats
path: root/src/bootstrap
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--src/bootstrap/CHANGELOG.md62
-rw-r--r--src/bootstrap/Cargo.lock778
-rw-r--r--src/bootstrap/Cargo.toml85
-rw-r--r--src/bootstrap/README.md333
-rw-r--r--src/bootstrap/bin/llvm-config-wrapper.rs24
-rw-r--r--src/bootstrap/bin/main.rs100
-rw-r--r--src/bootstrap/bin/rustc.rs369
-rw-r--r--src/bootstrap/bin/rustdoc.rs81
-rw-r--r--src/bootstrap/bin/sccache-plus-cl.rs38
-rw-r--r--src/bootstrap/bootstrap.py945
-rw-r--r--src/bootstrap/bootstrap_test.py87
-rw-r--r--src/bootstrap/build.rs43
-rw-r--r--src/bootstrap/builder.rs2312
-rw-r--r--src/bootstrap/builder/tests.rs671
-rw-r--r--src/bootstrap/cache.rs273
-rw-r--r--src/bootstrap/cc_detect.rs252
-rw-r--r--src/bootstrap/channel.rs105
-rw-r--r--src/bootstrap/check.rs498
-rw-r--r--src/bootstrap/clean.rs118
-rw-r--r--src/bootstrap/compile.rs1571
-rw-r--r--src/bootstrap/config.rs1668
-rwxr-xr-xsrc/bootstrap/configure.py493
-rw-r--r--src/bootstrap/defaults/README.md12
-rw-r--r--src/bootstrap/defaults/config.codegen.toml19
-rw-r--r--src/bootstrap/defaults/config.compiler.toml18
-rw-r--r--src/bootstrap/defaults/config.library.toml14
-rw-r--r--src/bootstrap/defaults/config.tools.toml22
-rw-r--r--src/bootstrap/defaults/config.user.toml9
-rw-r--r--src/bootstrap/dist.rs2157
-rw-r--r--src/bootstrap/doc.rs932
-rw-r--r--src/bootstrap/download-ci-llvm-stamp4
-rw-r--r--src/bootstrap/dylib_util.rs28
-rw-r--r--src/bootstrap/flags.rs817
-rw-r--r--src/bootstrap/format.rs175
-rw-r--r--src/bootstrap/install.rs292
-rw-r--r--src/bootstrap/job.rs140
-rw-r--r--src/bootstrap/lib.rs1679
-rw-r--r--src/bootstrap/metadata.rs59
-rw-r--r--src/bootstrap/metrics.rs208
-rw-r--r--src/bootstrap/mk/Makefile.in83
-rw-r--r--src/bootstrap/native.rs1335
-rw-r--r--src/bootstrap/run.rs105
-rw-r--r--src/bootstrap/sanity.rs242
-rw-r--r--src/bootstrap/setup.rs350
-rw-r--r--src/bootstrap/tarball.rs374
-rw-r--r--src/bootstrap/test.rs2542
-rw-r--r--src/bootstrap/tool.rs918
-rw-r--r--src/bootstrap/toolstate.rs479
-rw-r--r--src/bootstrap/util.rs603
49 files changed, 24522 insertions, 0 deletions
diff --git a/src/bootstrap/CHANGELOG.md b/src/bootstrap/CHANGELOG.md
new file mode 100644
index 000000000..85afc1f5f
--- /dev/null
+++ b/src/bootstrap/CHANGELOG.md
@@ -0,0 +1,62 @@
+# Changelog
+
+All notable changes to bootstrap will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
+
+
+## [Changes since the last major version]
+
+- Vendoring is no longer done automatically when building from git sources. To use vendoring, run `cargo vendor` manually, or use the pre-vendored `rustc-src` tarball.
+- `llvm-libunwind` now accepts `in-tree` (formerly true), `system` or `no` (formerly false) [#77703](https://github.com/rust-lang/rust/pull/77703)
+- The options `infodir`, `localstatedir`, and `gpg-password-file` are no longer allowed in config.toml. Previously, they were ignored without warning. Note that `infodir` and `localstatedir` are still accepted by `./configure`, with a warning. [#82451](https://github.com/rust-lang/rust/pull/82451)
+- Change the names for `dist` commands to match the component they generate. [#90684](https://github.com/rust-lang/rust/pull/90684)
+- The `build.fast-submodules` option has been removed. Fast submodule checkouts are enabled unconditionally. Automatic submodule handling can still be disabled with `build.submodules = false`.
+- Several unsupported `./configure` options have been removed: `optimize`, `parallel-compiler`. These can still be enabled with `--set`, although it isn't recommended.
+
+### Non-breaking changes
+
+- `x.py check` needs opt-in to check tests (--all-targets) [#77473](https://github.com/rust-lang/rust/pull/77473)
+- The default bootstrap profiles are now located at `bootstrap/defaults/config.$PROFILE.toml` (previously they were located at `bootstrap/defaults/config.toml.$PROFILE`) [#77558](https://github.com/rust-lang/rust/pull/77558)
+- If you have Rust already installed, `x.py` will now infer the host target
+ from the default rust toolchain. [#78513](https://github.com/rust-lang/rust/pull/78513)
+- Add options for enabling overflow checks, one for std (`overflow-checks-std`) and one for everything else (`overflow-checks`). Both default to false.
+
+
+## [Version 2] - 2020-09-25
+
+- `host` now defaults to the value of `build` in all cases
+ + Previously `host` defaulted to an empty list when `target` was overridden, and to `build` otherwise
+
+### Non-breaking changes
+
+- Add `x.py setup` [#76631](https://github.com/rust-lang/rust/pull/76631)
+- Add a changelog for x.py [#76626](https://github.com/rust-lang/rust/pull/76626)
+- Optionally, download LLVM from CI on Linux and NixOS. This can be enabled with `download-ci-llvm = true` under `[llvm]`.
+ + [#76439](https://github.com/rust-lang/rust/pull/76349)
+ + [#76667](https://github.com/rust-lang/rust/pull/76667)
+ + [#76708](https://github.com/rust-lang/rust/pull/76708)
+- Distribute rustc sources as part of `rustc-dev` [#76856](https://github.com/rust-lang/rust/pull/76856)
+- Make the default stage for x.py configurable [#76625](https://github.com/rust-lang/rust/pull/76625). This can be enabled with `build-stage = N`, `doc-stage = N`, etc.
+- Add a dedicated debug-logging option [#76588](https://github.com/rust-lang/rust/pull/76588). Previously, `debug-logging` could only be set with `debug-assertions`, slowing down the compiler more than necessary.
+- Add sample defaults for x.py [#76628](https://github.com/rust-lang/rust/pull/76628)
+- Add `--keep-stage-std`, which behaves like `keep-stage` but allows the stage
+ 0 compiler artifacts (i.e., stage1/bin/rustc) to be rebuilt if changed
+ [#77120](https://github.com/rust-lang/rust/pull/77120).
+
+
+## [Version 1] - 2020-09-11
+
+This is the first changelog entry, and it does not attempt to be an exhaustive list of features in x.py.
+Instead, this documents the changes to bootstrap in the past 2 months.
+
+- Improve defaults in `x.py` [#73964](https://github.com/rust-lang/rust/pull/73964)
+ (see [blog post] for details)
+- Set `ninja = true` by default [#74922](https://github.com/rust-lang/rust/pull/74922)
+- Avoid trying to inversely cross-compile for build triple from host triples [#76415](https://github.com/rust-lang/rust/pull/76415)
+- Allow blessing expect-tests in tools [#75975](https://github.com/rust-lang/rust/pull/75975)
+- `x.py check` checks tests/examples/benches [#76258](https://github.com/rust-lang/rust/pull/76258)
+- Fix `rust.use-lld` when linker is not set [#76326](https://github.com/rust-lang/rust/pull/76326)
+- Build tests with LLD if `use-lld = true` was passed [#76378](https://github.com/rust-lang/rust/pull/76378)
+
+[blog post]: https://blog.rust-lang.org/inside-rust/2020/08/30/changes-to-x-py-defaults.html
diff --git a/src/bootstrap/Cargo.lock b/src/bootstrap/Cargo.lock
new file mode 100644
index 000000000..664ffa1dd
--- /dev/null
+++ b/src/bootstrap/Cargo.lock
@@ -0,0 +1,778 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "aho-corasick"
+version = "0.7.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "ansi_term"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
+[[package]]
+name = "block-buffer"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324"
+dependencies = [
+ "generic-array",
+]
+
+[[package]]
+name = "bootstrap"
+version = "0.0.0"
+dependencies = [
+ "cc",
+ "cmake",
+ "fd-lock",
+ "filetime",
+ "getopts",
+ "hex",
+ "ignore",
+ "libc",
+ "num_cpus",
+ "once_cell",
+ "opener",
+ "pretty_assertions",
+ "serde",
+ "serde_json",
+ "sha2",
+ "sysinfo",
+ "tar",
+ "toml",
+ "walkdir",
+ "winapi",
+ "xz2",
+]
+
+[[package]]
+name = "bstr"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"
+dependencies = [
+ "lazy_static",
+ "memchr",
+ "regex-automata",
+]
+
+[[package]]
+name = "cc"
+version = "1.0.73"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "cmake"
+version = "0.1.48"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8ad8cef104ac57b68b89df3208164d228503abbdce70f6880ffa3d970e7443a"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
+
+[[package]]
+name = "cpufeatures"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "crossbeam-channel"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53"
+dependencies = [
+ "cfg-if",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
+dependencies = [
+ "cfg-if",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c"
+dependencies = [
+ "autocfg",
+ "cfg-if",
+ "crossbeam-utils",
+ "lazy_static",
+ "memoffset",
+ "scopeguard",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38"
+dependencies = [
+ "cfg-if",
+ "lazy_static",
+]
+
+[[package]]
+name = "crypto-common"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8"
+dependencies = [
+ "generic-array",
+ "typenum",
+]
+
+[[package]]
+name = "ctor"
+version = "0.1.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f877be4f7c9f246b183111634f75baa039715e3f46ce860677d3b19a69fb229c"
+dependencies = [
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "diff"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499"
+
+[[package]]
+name = "digest"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506"
+dependencies = [
+ "block-buffer",
+ "crypto-common",
+]
+
+[[package]]
+name = "either"
+version = "1.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
+
+[[package]]
+name = "errno"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1"
+dependencies = [
+ "errno-dragonfly",
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "errno-dragonfly"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
+dependencies = [
+ "cc",
+ "libc",
+]
+
+[[package]]
+name = "fd-lock"
+version = "3.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e11dcc7e4d79a8c89b9ab4c6f5c30b1fc4a83c420792da3542fd31179ed5f517"
+dependencies = [
+ "cfg-if",
+ "rustix",
+ "windows-sys",
+]
+
+[[package]]
+name = "filetime"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0408e2626025178a6a7f7ffc05a25bc47103229f19c113755de7bf63816290c"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall",
+ "winapi",
+]
+
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
+[[package]]
+name = "generic-array"
+version = "0.14.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803"
+dependencies = [
+ "typenum",
+ "version_check",
+]
+
+[[package]]
+name = "getopts"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5"
+dependencies = [
+ "unicode-width",
+]
+
+[[package]]
+name = "globset"
+version = "0.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10463d9ff00a2a068db14231982f5132edebad0d7660cd956a1c30292dbcbfbd"
+dependencies = [
+ "aho-corasick",
+ "bstr",
+ "fnv",
+ "log",
+ "regex",
+]
+
+[[package]]
+name = "hermit-abi"
+version = "0.1.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "hex"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
+
+[[package]]
+name = "ignore"
+version = "0.4.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "713f1b139373f96a2e0ce3ac931cd01ee973c3c5dd7c40c0c2efe96ad2b6751d"
+dependencies = [
+ "crossbeam-utils",
+ "globset",
+ "lazy_static",
+ "log",
+ "memchr",
+ "regex",
+ "same-file",
+ "thread_local",
+ "walkdir",
+ "winapi-util",
+]
+
+[[package]]
+name = "io-lifetimes"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24c3f4eff5495aee4c0399d7b6a0dc2b6e81be84242ffbfcf253ebacccc1d0cb"
+
+[[package]]
+name = "itoa"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d"
+
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+
+[[package]]
+name = "libc"
+version = "0.2.126"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836"
+
+[[package]]
+name = "linux-raw-sys"
+version = "0.0.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d"
+
+[[package]]
+name = "log"
+version = "0.4.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "lzma-sys"
+version = "0.1.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bdb4b7c3eddad11d3af9e86c487607d2d2442d185d848575365c4856ba96d619"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+]
+
+[[package]]
+name = "memchr"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
+
+[[package]]
+name = "memoffset"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "ntapi"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "num_cpus"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"
+dependencies = [
+ "hermit-abi",
+ "libc",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225"
+
+[[package]]
+name = "opener"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ea3ebcd72a54701f56345f16785a6d3ac2df7e986d273eb4395c0b01db17952"
+dependencies = [
+ "bstr",
+ "winapi",
+]
+
+[[package]]
+name = "output_vt100"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "628223faebab4e3e40667ee0b2336d34a5b960ff60ea743ddfdbcf7770bcfb66"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "pkg-config"
+version = "0.3.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"
+
+[[package]]
+name = "pretty_assertions"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1cab0e7c02cf376875e9335e0ba1da535775beb5450d21e1dffca068818ed98b"
+dependencies = [
+ "ansi_term",
+ "ctor",
+ "diff",
+ "output_vt100",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.39"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rayon"
+version = "1.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d"
+dependencies = [
+ "autocfg",
+ "crossbeam-deque",
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f"
+dependencies = [
+ "crossbeam-channel",
+ "crossbeam-deque",
+ "crossbeam-utils",
+ "num_cpus",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.2.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42"
+dependencies = [
+ "bitflags",
+]
+
+[[package]]
+name = "regex"
+version = "1.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
+
+[[package]]
+name = "regex-syntax"
+version = "0.6.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64"
+
+[[package]]
+name = "rustix"
+version = "0.35.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef258c11e17f5c01979a10543a30a4e12faef6aab217a74266e747eefa3aed88"
+dependencies = [
+ "bitflags",
+ "errno",
+ "io-lifetimes",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys",
+]
+
+[[package]]
+name = "ryu"
+version = "1.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695"
+
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
+[[package]]
+name = "scopeguard"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
+
+[[package]]
+name = "serde"
+version = "1.0.137"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.137"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.81"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "sha2"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
+[[package]]
+name = "syn"
+version = "1.0.95"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fbaf6116ab8924f39d52792136fb74fd60a80194cf1b1c6ffa6453eef1c3f942"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "sysinfo"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a2809487b962344ca55d9aea565f9ffbcb6929780802217acc82561f6746770"
+dependencies = [
+ "cfg-if",
+ "core-foundation-sys",
+ "libc",
+ "ntapi",
+ "once_cell",
+ "rayon",
+ "winapi",
+]
+
+[[package]]
+name = "tar"
+version = "0.4.38"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6"
+dependencies = [
+ "filetime",
+ "libc",
+ "xattr",
+]
+
+[[package]]
+name = "thread_local"
+version = "1.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180"
+dependencies = [
+ "once_cell",
+]
+
+[[package]]
+name = "toml"
+version = "0.5.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "typenum"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee"
+
+[[package]]
+name = "unicode-width"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
+
+[[package]]
+name = "version_check"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
+
+[[package]]
+name = "walkdir"
+version = "2.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56"
+dependencies = [
+ "same-file",
+ "winapi",
+ "winapi-util",
+]
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-util"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
+[[package]]
+name = "windows-sys"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
+dependencies = [
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
+
+[[package]]
+name = "xattr"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "xz2"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c179869f34fc7c01830d3ce7ea2086bc3a07e0d35289b667d0a8bf910258926c"
+dependencies = [
+ "lzma-sys",
+]
diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml
new file mode 100644
index 000000000..84f6aaf99
--- /dev/null
+++ b/src/bootstrap/Cargo.toml
@@ -0,0 +1,85 @@
+[package]
+name = "bootstrap"
+version = "0.0.0"
+edition = "2021"
+build = "build.rs"
+default-run = "bootstrap"
+
+[lib]
+path = "lib.rs"
+doctest = false
+
+[[bin]]
+name = "bootstrap"
+path = "bin/main.rs"
+test = false
+
+[[bin]]
+name = "rustc"
+path = "bin/rustc.rs"
+test = false
+
+[[bin]]
+name = "rustdoc"
+path = "bin/rustdoc.rs"
+test = false
+
+[[bin]]
+name = "sccache-plus-cl"
+path = "bin/sccache-plus-cl.rs"
+test = false
+
+[[bin]]
+name = "llvm-config-wrapper"
+path = "bin/llvm-config-wrapper.rs"
+test = false
+
+[dependencies]
+cmake = "0.1.38"
+fd-lock = "3.0.6"
+filetime = "0.2"
+num_cpus = "1.0"
+getopts = "0.2.19"
+cc = "1.0.69"
+libc = "0.2"
+hex = "0.4"
+serde = { version = "1.0.8", features = ["derive"] }
+serde_json = "1.0.2"
+sha2 = "0.10"
+tar = "0.4"
+toml = "0.5"
+ignore = "0.4.10"
+opener = "0.5"
+once_cell = "1.7.2"
+xz2 = "0.1"
+walkdir = "2"
+
+# Dependencies needed by the build-metrics feature
+sysinfo = { version = "0.24.1", optional = true }
+
+[target.'cfg(windows)'.dependencies.winapi]
+version = "0.3"
+features = [
+ "fileapi",
+ "ioapiset",
+ "jobapi2",
+ "handleapi",
+ "winioctl",
+ "psapi",
+ "impl-default",
+ "timezoneapi",
+]
+
+[dev-dependencies]
+pretty_assertions = "0.7"
+
+[features]
+build-metrics = ["sysinfo"]
+
+# We care a lot about bootstrap's compile times, so don't include debuginfo for
+# dependencies, only bootstrap itself.
+[profile.dev]
+debug = 0
+[profile.dev.package]
+# Only use debuginfo=1 to further reduce compile times.
+bootstrap.debug = 1
diff --git a/src/bootstrap/README.md b/src/bootstrap/README.md
new file mode 100644
index 000000000..a2e596bf4
--- /dev/null
+++ b/src/bootstrap/README.md
@@ -0,0 +1,333 @@
+# rustbuild - Bootstrapping Rust
+
+This is an in-progress README which is targeted at helping to explain how Rust
+is bootstrapped and in general some of the technical details of the build
+system.
+
+## Using rustbuild
+
+The rustbuild build system has a primary entry point, a top level `x.py` script:
+
+```sh
+$ python ./x.py build
+```
+
+Note that if you're on Unix you should be able to execute the script directly:
+
+```sh
+$ ./x.py build
+```
+
+The script accepts commands, flags, and arguments to determine what to do:
+
+* `build` - a general purpose command for compiling code. Alone `build` will
+ bootstrap the entire compiler, and otherwise arguments passed indicate what to
+ build. For example:
+
+ ```
+ # build the whole compiler
+ ./x.py build --stage 2
+
+ # build the stage1 compiler
+ ./x.py build
+
+ # build stage0 libstd
+ ./x.py build --stage 0 library/std
+
+ # build a particular crate in stage0
+ ./x.py build --stage 0 library/test
+ ```
+
+ If files are dirty that would normally be rebuilt from stage 0, that can be
+ overridden using `--keep-stage 0`. Using `--keep-stage n` will skip all steps
+ that belong to stage n or earlier:
+
+ ```
+ # build stage 1, keeping old build products for stage 0
+ ./x.py build --keep-stage 0
+ ```
+
+* `test` - a command for executing unit tests. Like the `build` command this
+ will execute the entire test suite by default, and otherwise it can be used to
+ select which test suite is run:
+
+ ```
+ # run all unit tests
+ ./x.py test
+
+ # execute tool tests
+ ./x.py test tidy
+
+ # execute the UI test suite
+ ./x.py test src/test/ui
+
+ # execute only some tests in the UI test suite
+ ./x.py test src/test/ui --test-args substring-of-test-name
+
+ # execute tests in the standard library in stage0
+ ./x.py test --stage 0 library/std
+
+ # execute tests in the core and standard library in stage0,
+ # without running doc tests (thus avoid depending on building the compiler)
+ ./x.py test --stage 0 --no-doc library/core library/std
+
+ # execute all doc tests
+ ./x.py test src/doc
+ ```
+
+* `doc` - a command for building documentation. Like above can take arguments
+ for what to document.
+
+## Configuring rustbuild
+
+There are currently two methods for configuring the rustbuild build system.
+
+First, rustbuild offers a TOML-based configuration system with a `config.toml`
+file. An example of this configuration can be found at `config.toml.example`,
+and the configuration file can also be passed as `--config path/to/config.toml`
+if the build system is being invoked manually (via the python script).
+
+Next, the `./configure` options serialized in `config.mk` will be
+parsed and read. That is, if any `./configure` options are passed, they'll be
+handled naturally. `./configure` should almost never be used for local
+installations, and is primarily useful for CI. Prefer to customize behavior
+using `config.toml`.
+
+Finally, rustbuild makes use of the [cc-rs crate] which has [its own
+method][env-vars] of configuring C compilers and C flags via environment
+variables.
+
+[cc-rs crate]: https://github.com/alexcrichton/cc-rs
+[env-vars]: https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables
+
+## Build stages
+
+The rustbuild build system goes through a few phases to actually build the
+compiler. What actually happens when you invoke rustbuild is:
+
+1. The entry point script, `x.py` is run. This script is
+ responsible for downloading the stage0 compiler/Cargo binaries, and it then
+ compiles the build system itself (this folder). Finally, it then invokes the
+ actual `bootstrap` binary build system.
+2. In Rust, `bootstrap` will slurp up all configuration, perform a number of
+ sanity checks (compilers exist for example), and then start building the
+ stage0 artifacts.
+3. The stage0 `cargo` downloaded earlier is used to build the standard library
+ and the compiler, and then these binaries are then copied to the `stage1`
+ directory. That compiler is then used to generate the stage1 artifacts which
+ are then copied to the stage2 directory, and then finally the stage2
+ artifacts are generated using that compiler.
+
+The goal of each stage is to (a) leverage Cargo as much as possible and failing
+that (b) leverage Rust as much as possible!
+
+## Incremental builds
+
+You can configure rustbuild to use incremental compilation with the
+`--incremental` flag:
+
+```sh
+$ ./x.py build --incremental
+```
+
+The `--incremental` flag will store incremental compilation artifacts
+in `build/<host>/stage0-incremental`. Note that we only use incremental
+compilation for the stage0 -> stage1 compilation -- this is because
+the stage1 compiler is changing, and we don't try to cache and reuse
+incremental artifacts across different versions of the compiler.
+
+You can always drop the `--incremental` to build as normal (but you
+will still be using the local nightly as your bootstrap).
+
+## Directory Layout
+
+This build system houses all output under the `build` directory, which looks
+like this:
+
+```sh
+# Root folder of all output. Everything is scoped underneath here
+build/
+
+ # Location where the stage0 compiler downloads are all cached. This directory
+ # only contains the tarballs themselves as they're extracted elsewhere.
+ cache/
+ 2015-12-19/
+ 2016-01-15/
+ 2016-01-21/
+ ...
+
+ # Output directory for building this build system itself. The stage0
+ # cargo/rustc are used to build the build system into this location.
+ bootstrap/
+ debug/
+ release/
+
+ # Output of the dist-related steps like dist-std, dist-rustc, and dist-docs
+ dist/
+
+ # Temporary directory used for various input/output as part of various stages
+ tmp/
+
+ # Each remaining directory is scoped by the "host" triple of compilation at
+ # hand.
+ x86_64-unknown-linux-gnu/
+
+ # The build artifacts for the `compiler-rt` library for the target this
+ # folder is under. The exact layout here will likely depend on the platform,
+ # and this is also built with CMake so the build system is also likely
+ # different.
+ compiler-rt/
+ build/
+
+ # Output folder for LLVM if it is compiled for this target
+ llvm/
+
+ # build folder (e.g. the platform-specific build system). Like with
+ # compiler-rt this is compiled with CMake
+ build/
+
+ # Installation of LLVM. Note that we run the equivalent of 'make install'
+ # for LLVM to setup these folders.
+ bin/
+ lib/
+ include/
+ share/
+ ...
+
+ # Output folder for all documentation of this target. This is what's filled
+ # in whenever the `doc` step is run.
+ doc/
+
+ # Output for all compiletest-based test suites
+ test/
+ ui/
+ debuginfo/
+ ...
+
+ # Location where the stage0 Cargo and Rust compiler are unpacked. This
+ # directory is purely an extracted and overlaid tarball of these two (done
+ # by the bootstrapy python script). In theory the build system does not
+ # modify anything under this directory afterwards.
+ stage0/
+
+ # These to build directories are the cargo output directories for builds of
+ # the standard library and compiler, respectively. Internally these may also
+ # have other target directories, which represent artifacts being compiled
+ # from the host to the specified target.
+ #
+ # Essentially, each of these directories is filled in by one `cargo`
+ # invocation. The build system instruments calling Cargo in the right order
+ # with the right variables to ensure these are filled in correctly.
+ stageN-std/
+ stageN-test/
+ stageN-rustc/
+ stageN-tools/
+
+ # This is a special case of the above directories, **not** filled in via
+ # Cargo but rather the build system itself. The stage0 compiler already has
+ # a set of target libraries for its own host triple (in its own sysroot)
+ # inside of stage0/. When we run the stage0 compiler to bootstrap more
+ # things, however, we don't want to use any of these libraries (as those are
+ # the ones that we're building). So essentially, when the stage1 compiler is
+ # being compiled (e.g. after libstd has been built), *this* is used as the
+ # sysroot for the stage0 compiler being run.
+ #
+ # Basically this directory is just a temporary artifact use to configure the
+ # stage0 compiler to ensure that the libstd we just built is used to
+ # compile the stage1 compiler.
+ stage0-sysroot/lib/
+
+ # These output directories are intended to be standalone working
+ # implementations of the compiler (corresponding to each stage). The build
+ # system will link (using hard links) output from stageN-{std,rustc} into
+ # each of these directories.
+ #
+ # In theory there is no extra build output in these directories.
+ stage1/
+ stage2/
+ stage3/
+```
+
+## Cargo projects
+
+The current build is unfortunately not quite as simple as `cargo build` in a
+directory, but rather the compiler is split into three different Cargo projects:
+
+* `library/std` - the standard library
+* `library/test` - testing support, depends on libstd
+* `compiler/rustc` - the actual compiler itself
+
+Each "project" has a corresponding Cargo.lock file with all dependencies, and
+this means that building the compiler involves running Cargo three times. The
+structure here serves two goals:
+
+1. Facilitating dependencies coming from crates.io. These dependencies don't
+ depend on `std`, so libstd is a separate project compiled ahead of time
+ before the actual compiler builds.
+2. Splitting "host artifacts" from "target artifacts". That is, when building
+ code for an arbitrary target you don't need the entire compiler, but you'll
+ end up needing libraries like libtest that depend on std but also want to use
+ crates.io dependencies. Hence, libtest is split out as its own project that
+ is sequenced after `std` but before `rustc`. This project is built for all
+ targets.
+
+There is some loss in build parallelism here because libtest can be compiled in
+parallel with a number of rustc artifacts, but in theory the loss isn't too bad!
+
+## Build tools
+
+We've actually got quite a few tools that we use in the compiler's build system
+and for testing. To organize these, each tool is a project in `src/tools` with a
+corresponding `Cargo.toml`. All tools are compiled with Cargo (currently having
+independent `Cargo.lock` files) and do not currently explicitly depend on the
+compiler or standard library. Compiling each tool is sequenced after the
+appropriate libstd/libtest/librustc compile above.
+
+## Extending rustbuild
+
+So you'd like to add a feature to the rustbuild build system or just fix a bug.
+Great! One of the major motivational factors for moving away from `make` is that
+Rust is in theory much easier to read, modify, and write. If you find anything
+excessively confusing, please open an issue on this and we'll try to get it
+documented or simplified pronto.
+
+First up, you'll probably want to read over the documentation above as that'll
+give you a high level overview of what rustbuild is doing. You also probably
+want to play around a bit yourself by just getting it up and running before you
+dive too much into the actual build system itself.
+
+After that, each module in rustbuild should have enough documentation to keep
+you up and running. Some general areas that you may be interested in modifying
+are:
+
+* Adding a new build tool? Take a look at `bootstrap/tool.rs` for examples of
+ other tools.
+* Adding a new compiler crate? Look no further! Adding crates can be done by
+ adding a new directory with `Cargo.toml` followed by configuring all
+ `Cargo.toml` files accordingly.
+* Adding a new dependency from crates.io? This should just work inside the
+ compiler artifacts stage (everything other than libtest and libstd).
+* Adding a new configuration option? You'll want to modify `bootstrap/flags.rs`
+ for command line flags and then `bootstrap/config.rs` to copy the flags to the
+ `Config` struct.
+* Adding a sanity check? Take a look at `bootstrap/sanity.rs`.
+
+If you make a major change, please remember to:
+
++ Update `VERSION` in `src/bootstrap/main.rs`.
+* Update `changelog-seen = N` in `config.toml.example`.
+* Add an entry in `src/bootstrap/CHANGELOG.md`.
+
+A 'major change' includes
+
+* A new option or
+* A change in the default options.
+
+Changes that do not affect contributors to the compiler or users
+building rustc from source don't need an update to `VERSION`.
+
+If you have any questions feel free to reach out on the `#t-infra` channel in
+the [Rust Zulip server][rust-zulip] or ask on internals.rust-lang.org. When
+you encounter bugs, please file issues on the rust-lang/rust issue tracker.
+
+[rust-zulip]: https://rust-lang.zulipchat.com/#narrow/stream/242791-t-infra
diff --git a/src/bootstrap/bin/llvm-config-wrapper.rs b/src/bootstrap/bin/llvm-config-wrapper.rs
new file mode 100644
index 000000000..89984bb55
--- /dev/null
+++ b/src/bootstrap/bin/llvm-config-wrapper.rs
@@ -0,0 +1,24 @@
+// The sheer existence of this file is an awful hack. See the comments in
+// `src/bootstrap/native.rs` for why this is needed when compiling LLD.
+
+use std::env;
+use std::io::{self, Write};
+use std::process::{self, Command, Stdio};
+
+fn main() {
+ let real_llvm_config = env::var_os("LLVM_CONFIG_REAL").unwrap();
+ let mut cmd = Command::new(real_llvm_config);
+ cmd.args(env::args().skip(1)).stderr(Stdio::piped());
+ let output = cmd.output().expect("failed to spawn llvm-config");
+ let mut stdout = String::from_utf8_lossy(&output.stdout);
+
+ if let Ok(to_replace) = env::var("LLVM_CONFIG_SHIM_REPLACE") {
+ if let Ok(replace_with) = env::var("LLVM_CONFIG_SHIM_REPLACE_WITH") {
+ stdout = stdout.replace(&to_replace, &replace_with).into();
+ }
+ }
+
+ print!("{}", stdout.replace("\\", "/"));
+ io::stdout().flush().unwrap();
+ process::exit(output.status.code().unwrap_or(1));
+}
diff --git a/src/bootstrap/bin/main.rs b/src/bootstrap/bin/main.rs
new file mode 100644
index 000000000..9b4861ccd
--- /dev/null
+++ b/src/bootstrap/bin/main.rs
@@ -0,0 +1,100 @@
+//! rustbuild, the Rust build system
+//!
+//! This is the entry point for the build system used to compile the `rustc`
+//! compiler. Lots of documentation can be found in the `README.md` file in the
+//! parent directory, and otherwise documentation can be found throughout the `build`
+//! directory in each respective module.
+
+use std::env;
+
+use bootstrap::{t, Build, Config, Subcommand, VERSION};
+
+fn main() {
+ let args = env::args().skip(1).collect::<Vec<_>>();
+ let config = Config::parse(&args);
+
+ let mut build_lock;
+ let _build_lock_guard;
+ if cfg!(any(unix, windows)) {
+ build_lock = fd_lock::RwLock::new(t!(std::fs::File::create(config.out.join("lock"))));
+ _build_lock_guard = match build_lock.try_write() {
+ Ok(lock) => lock,
+ err => {
+ println!("warning: build directory locked, waiting for lock");
+ drop(err);
+ t!(build_lock.write())
+ }
+ };
+ } else {
+ println!("warning: file locking not supported for target, not locking build directory");
+ }
+
+ // check_version warnings are not printed during setup
+ let changelog_suggestion =
+ if matches!(config.cmd, Subcommand::Setup { .. }) { None } else { check_version(&config) };
+
+ // NOTE: Since `./configure` generates a `config.toml`, distro maintainers will see the
+ // changelog warning, not the `x.py setup` message.
+ let suggest_setup = !config.config.exists() && !matches!(config.cmd, Subcommand::Setup { .. });
+ if suggest_setup {
+ println!("warning: you have not made a `config.toml`");
+ println!(
+ "help: consider running `./x.py setup` or copying `config.toml.example` by running \
+ `cp config.toml.example config.toml`"
+ );
+ } else if let Some(suggestion) = &changelog_suggestion {
+ println!("{}", suggestion);
+ }
+
+ let pre_commit = config.src.join(".git").join("hooks").join("pre-commit");
+ Build::new(config).build();
+
+ if suggest_setup {
+ println!("warning: you have not made a `config.toml`");
+ println!(
+ "help: consider running `./x.py setup` or copying `config.toml.example` by running \
+ `cp config.toml.example config.toml`"
+ );
+ } else if let Some(suggestion) = &changelog_suggestion {
+ println!("{}", suggestion);
+ }
+
+ // Give a warning if the pre-commit script is in pre-commit and not pre-push.
+ // HACK: Since the commit script uses hard links, we can't actually tell if it was installed by x.py setup or not.
+ // We could see if it's identical to src/etc/pre-push.sh, but pre-push may have been modified in the meantime.
+ // Instead, look for this comment, which is almost certainly not in any custom hook.
+ if std::fs::read_to_string(pre_commit).map_or(false, |contents| {
+ contents.contains("https://github.com/rust-lang/rust/issues/77620#issuecomment-705144570")
+ }) {
+ println!(
+ "warning: You have the pre-push script installed to .git/hooks/pre-commit. \
+ Consider moving it to .git/hooks/pre-push instead, which runs less often."
+ );
+ }
+
+ if suggest_setup || changelog_suggestion.is_some() {
+ println!("note: this message was printed twice to make it more likely to be seen");
+ }
+}
+
+fn check_version(config: &Config) -> Option<String> {
+ let mut msg = String::new();
+
+ let suggestion = if let Some(seen) = config.changelog_seen {
+ if seen != VERSION {
+ msg.push_str("warning: there have been changes to x.py since you last updated.\n");
+ format!("update `config.toml` to use `changelog-seen = {}` instead", VERSION)
+ } else {
+ return None;
+ }
+ } else {
+ msg.push_str("warning: x.py has made several changes recently you may want to look at\n");
+ format!("add `changelog-seen = {}` at the top of `config.toml`", VERSION)
+ };
+
+ msg.push_str("help: consider looking at the changes in `src/bootstrap/CHANGELOG.md`\n");
+ msg.push_str("note: to silence this warning, ");
+ msg.push_str(&suggestion);
+
+ Some(msg)
+}
diff --git a/src/bootstrap/bin/rustc.rs b/src/bootstrap/bin/rustc.rs
new file mode 100644
index 000000000..40a3cc6d1
--- /dev/null
+++ b/src/bootstrap/bin/rustc.rs
@@ -0,0 +1,369 @@
+//! Shim which is passed to Cargo as "rustc" when running the bootstrap.
+//!
+//! This shim will take care of some various tasks that our build process
+//! requires that Cargo can't quite do through normal configuration:
+//!
+//! 1. When compiling build scripts and build dependencies, we need a guaranteed
+//! full standard library available. The only compiler which actually has
+//! this is the snapshot, so we detect this situation and always compile with
+//! the snapshot compiler.
+//! 2. We pass a bunch of `--cfg` and other flags based on what we're compiling
+//! (and this slightly differs based on a whether we're using a snapshot or
+//! not), so we do that all here.
+//!
+//! This may one day be replaced by RUSTFLAGS, but the dynamic nature of
+//! switching compilers for the bootstrap and for build scripts will probably
+//! never get replaced.
+
+include!("../dylib_util.rs");
+
+use std::env;
+use std::path::PathBuf;
+use std::process::{Child, Command};
+use std::str::FromStr;
+use std::time::Instant;
+
+fn main() {
+ let args = env::args_os().skip(1).collect::<Vec<_>>();
+
+ // Detect whether or not we're a build script depending on whether --target
+ // is passed (a bit janky...)
+ let target = args.windows(2).find(|w| &*w[0] == "--target").and_then(|w| w[1].to_str());
+ let version = args.iter().find(|w| &**w == "-vV");
+
+ let verbose = match env::var("RUSTC_VERBOSE") {
+ Ok(s) => usize::from_str(&s).expect("RUSTC_VERBOSE should be an integer"),
+ Err(_) => 0,
+ };
+
+ // Use a different compiler for build scripts, since there may not yet be a
+ // libstd for the real compiler to use. However, if Cargo is attempting to
+ // determine the version of the compiler, the real compiler needs to be
+ // used. Currently, these two states are differentiated based on whether
+ // --target and -vV is/isn't passed.
+ let (rustc, libdir) = if target.is_none() && version.is_none() {
+ ("RUSTC_SNAPSHOT", "RUSTC_SNAPSHOT_LIBDIR")
+ } else {
+ ("RUSTC_REAL", "RUSTC_LIBDIR")
+ };
+ let stage = env::var("RUSTC_STAGE").expect("RUSTC_STAGE was not set");
+ let sysroot = env::var_os("RUSTC_SYSROOT").expect("RUSTC_SYSROOT was not set");
+ let on_fail = env::var_os("RUSTC_ON_FAIL").map(Command::new);
+
+ let rustc = env::var_os(rustc).unwrap_or_else(|| panic!("{:?} was not set", rustc));
+ let libdir = env::var_os(libdir).unwrap_or_else(|| panic!("{:?} was not set", libdir));
+ let mut dylib_path = dylib_path();
+ dylib_path.insert(0, PathBuf::from(&libdir));
+
+ let mut cmd = Command::new(rustc);
+ cmd.args(&args).env(dylib_path_var(), env::join_paths(&dylib_path).unwrap());
+
+ // Get the name of the crate we're compiling, if any.
+ let crate_name =
+ args.windows(2).find(|args| args[0] == "--crate-name").and_then(|args| args[1].to_str());
+
+ if let Some(crate_name) = crate_name {
+ if let Some(target) = env::var_os("RUSTC_TIME") {
+ if target == "all"
+ || target.into_string().unwrap().split(',').any(|c| c.trim() == crate_name)
+ {
+ cmd.arg("-Ztime");
+ }
+ }
+ }
+
+ // Print backtrace in case of ICE
+ if env::var("RUSTC_BACKTRACE_ON_ICE").is_ok() && env::var("RUST_BACKTRACE").is_err() {
+ cmd.env("RUST_BACKTRACE", "1");
+ }
+
+ if let Ok(lint_flags) = env::var("RUSTC_LINT_FLAGS") {
+ cmd.args(lint_flags.split_whitespace());
+ }
+
+ if target.is_some() {
+ // The stage0 compiler has a special sysroot distinct from what we
+ // actually downloaded, so we just always pass the `--sysroot` option,
+ // unless one is already set.
+ if !args.iter().any(|arg| arg == "--sysroot") {
+ cmd.arg("--sysroot").arg(&sysroot);
+ }
+
+ // If we're compiling specifically the `panic_abort` crate then we pass
+ // the `-C panic=abort` option. Note that we do not do this for any
+ // other crate intentionally as this is the only crate for now that we
+ // ship with panic=abort.
+ //
+ // This... is a bit of a hack how we detect this. Ideally this
+ // information should be encoded in the crate I guess? Would likely
+ // require an RFC amendment to RFC 1513, however.
+ //
+ // `compiler_builtins` are unconditionally compiled with panic=abort to
+ // workaround undefined references to `rust_eh_unwind_resume` generated
+ // otherwise, see issue https://github.com/rust-lang/rust/issues/43095.
+ if crate_name == Some("panic_abort")
+ || crate_name == Some("compiler_builtins") && stage != "0"
+ {
+ cmd.arg("-C").arg("panic=abort");
+ }
+ } else {
+ // FIXME(rust-lang/cargo#5754) we shouldn't be using special env vars
+ // here, but rather Cargo should know what flags to pass rustc itself.
+
+ // Override linker if necessary.
+ if let Ok(host_linker) = env::var("RUSTC_HOST_LINKER") {
+ cmd.arg(format!("-Clinker={}", host_linker));
+ }
+ if env::var_os("RUSTC_HOST_FUSE_LD_LLD").is_some() {
+ cmd.arg("-Clink-args=-fuse-ld=lld");
+ }
+
+ if let Ok(s) = env::var("RUSTC_HOST_CRT_STATIC") {
+ if s == "true" {
+ cmd.arg("-C").arg("target-feature=+crt-static");
+ }
+ if s == "false" {
+ cmd.arg("-C").arg("target-feature=-crt-static");
+ }
+ }
+
+ // Cargo doesn't pass RUSTFLAGS to proc_macros:
+ // https://github.com/rust-lang/cargo/issues/4423
+ // Thus, if we are on stage 0, we explicitly set `--cfg=bootstrap`.
+ // We also declare that the flag is expected, which is mainly needed for
+ // later stages so that they don't warn about #[cfg(bootstrap)],
+ // but enabling it for stage 0 too lets any warnings, if they occur,
+ // occur more early on, e.g. about #[cfg(bootstrap = "foo")].
+ if stage == "0" {
+ cmd.arg("--cfg=bootstrap");
+ }
+ cmd.arg("-Zunstable-options");
+ cmd.arg("--check-cfg=values(bootstrap)");
+ }
+
+ if let Ok(map) = env::var("RUSTC_DEBUGINFO_MAP") {
+ cmd.arg("--remap-path-prefix").arg(&map);
+ }
+
+ // Force all crates compiled by this compiler to (a) be unstable and (b)
+ // allow the `rustc_private` feature to link to other unstable crates
+ // also in the sysroot. We also do this for host crates, since those
+ // may be proc macros, in which case we might ship them.
+ if env::var_os("RUSTC_FORCE_UNSTABLE").is_some() && (stage != "0" || target.is_some()) {
+ cmd.arg("-Z").arg("force-unstable-if-unmarked");
+ }
+
+ if let Ok(flags) = env::var("MAGIC_EXTRA_RUSTFLAGS") {
+ for flag in flags.split(' ') {
+ cmd.arg(flag);
+ }
+ }
+
+ let is_test = args.iter().any(|a| a == "--test");
+ if verbose > 2 {
+ let rust_env_vars =
+ env::vars().filter(|(k, _)| k.starts_with("RUST") || k.starts_with("CARGO"));
+ let prefix = if is_test { "[RUSTC-SHIM] rustc --test" } else { "[RUSTC-SHIM] rustc" };
+ let prefix = match crate_name {
+ Some(crate_name) => format!("{} {}", prefix, crate_name),
+ None => prefix.to_string(),
+ };
+ for (i, (k, v)) in rust_env_vars.enumerate() {
+ eprintln!("{} env[{}]: {:?}={:?}", prefix, i, k, v);
+ }
+ eprintln!("{} working directory: {}", prefix, env::current_dir().unwrap().display());
+ eprintln!(
+ "{} command: {:?}={:?} {:?}",
+ prefix,
+ dylib_path_var(),
+ env::join_paths(&dylib_path).unwrap(),
+ cmd,
+ );
+ eprintln!("{} sysroot: {:?}", prefix, sysroot);
+ eprintln!("{} libdir: {:?}", prefix, libdir);
+ }
+
+ let start = Instant::now();
+ let (child, status) = {
+ let errmsg = format!("\nFailed to run:\n{:?}\n-------------", cmd);
+ let mut child = cmd.spawn().expect(&errmsg);
+ let status = child.wait().expect(&errmsg);
+ (child, status)
+ };
+
+ if env::var_os("RUSTC_PRINT_STEP_TIMINGS").is_some()
+ || env::var_os("RUSTC_PRINT_STEP_RUSAGE").is_some()
+ {
+ if let Some(crate_name) = crate_name {
+ let dur = start.elapsed();
+ // If the user requested resource usage data, then
+ // include that in addition to the timing output.
+ let rusage_data =
+ env::var_os("RUSTC_PRINT_STEP_RUSAGE").and_then(|_| format_rusage_data(child));
+ eprintln!(
+ "[RUSTC-TIMING] {} test:{} {}.{:03}{}{}",
+ crate_name,
+ is_test,
+ dur.as_secs(),
+ dur.subsec_millis(),
+ if rusage_data.is_some() { " " } else { "" },
+ rusage_data.unwrap_or(String::new()),
+ );
+ }
+ }
+
+ if status.success() {
+ std::process::exit(0);
+ // note: everything below here is unreachable. do not put code that
+ // should run on success, after this block.
+ }
+ if verbose > 0 {
+ println!("\nDid not run successfully: {}\n{:?}\n-------------", status, cmd);
+ }
+
+ if let Some(mut on_fail) = on_fail {
+ on_fail.status().expect("Could not run the on_fail command");
+ }
+
+ // Preserve the exit code. In case of signal, exit with 0xfe since it's
+ // awkward to preserve this status in a cross-platform way.
+ match status.code() {
+ Some(i) => std::process::exit(i),
+ None => {
+ eprintln!("rustc exited with {}", status);
+ std::process::exit(0xfe);
+ }
+ }
+}
+
+#[cfg(all(not(unix), not(windows)))]
+// In the future we can add this for more platforms
+fn format_rusage_data(_child: Child) -> Option<String> {
+ None
+}
+
+#[cfg(windows)]
+fn format_rusage_data(child: Child) -> Option<String> {
+ use std::os::windows::io::AsRawHandle;
+ use winapi::um::{processthreadsapi, psapi, timezoneapi};
+ let handle = child.as_raw_handle();
+ macro_rules! try_bool {
+ ($e:expr) => {
+ if $e != 1 {
+ return None;
+ }
+ };
+ }
+
+ let mut user_filetime = Default::default();
+ let mut user_time = Default::default();
+ let mut kernel_filetime = Default::default();
+ let mut kernel_time = Default::default();
+ let mut memory_counters = psapi::PROCESS_MEMORY_COUNTERS::default();
+
+ unsafe {
+ try_bool!(processthreadsapi::GetProcessTimes(
+ handle,
+ &mut Default::default(),
+ &mut Default::default(),
+ &mut kernel_filetime,
+ &mut user_filetime,
+ ));
+ try_bool!(timezoneapi::FileTimeToSystemTime(&user_filetime, &mut user_time));
+ try_bool!(timezoneapi::FileTimeToSystemTime(&kernel_filetime, &mut kernel_time));
+
+ // Unlike on Linux with RUSAGE_CHILDREN, this will only return memory information for the process
+ // with the given handle and none of that process's children.
+ try_bool!(psapi::GetProcessMemoryInfo(
+ handle as _,
+ &mut memory_counters as *mut _ as _,
+ std::mem::size_of::<psapi::PROCESS_MEMORY_COUNTERS_EX>() as u32,
+ ));
+ }
+
+ // Guide on interpreting these numbers:
+ // https://docs.microsoft.com/en-us/windows/win32/psapi/process-memory-usage-information
+ let peak_working_set = memory_counters.PeakWorkingSetSize / 1024;
+ let peak_page_file = memory_counters.PeakPagefileUsage / 1024;
+ let peak_paged_pool = memory_counters.QuotaPeakPagedPoolUsage / 1024;
+ let peak_nonpaged_pool = memory_counters.QuotaPeakNonPagedPoolUsage / 1024;
+ Some(format!(
+ "user: {USER_SEC}.{USER_USEC:03} \
+ sys: {SYS_SEC}.{SYS_USEC:03} \
+ peak working set (kb): {PEAK_WORKING_SET} \
+ peak page file usage (kb): {PEAK_PAGE_FILE} \
+ peak paged pool usage (kb): {PEAK_PAGED_POOL} \
+ peak non-paged pool usage (kb): {PEAK_NONPAGED_POOL} \
+ page faults: {PAGE_FAULTS}",
+ USER_SEC = user_time.wSecond + (user_time.wMinute * 60),
+ USER_USEC = user_time.wMilliseconds,
+ SYS_SEC = kernel_time.wSecond + (kernel_time.wMinute * 60),
+ SYS_USEC = kernel_time.wMilliseconds,
+ PEAK_WORKING_SET = peak_working_set,
+ PEAK_PAGE_FILE = peak_page_file,
+ PEAK_PAGED_POOL = peak_paged_pool,
+ PEAK_NONPAGED_POOL = peak_nonpaged_pool,
+ PAGE_FAULTS = memory_counters.PageFaultCount,
+ ))
+}
+
+#[cfg(unix)]
+/// Tries to build a string with human readable data for several of the rusage
+/// fields. Note that we are focusing mainly on data that we believe to be
+/// supplied on Linux (the `rusage` struct has other fields in it but they are
+/// currently unsupported by Linux).
+fn format_rusage_data(_child: Child) -> Option<String> {
+ let rusage: libc::rusage = unsafe {
+ let mut recv = std::mem::zeroed();
+ // -1 is RUSAGE_CHILDREN, which means to get the rusage for all children
+ // (and grandchildren, etc) processes that have respectively terminated
+ // and been waited for.
+ let retval = libc::getrusage(-1, &mut recv);
+ if retval != 0 {
+ return None;
+ }
+ recv
+ };
+ // Mac OS X reports the maxrss in bytes, not kb.
+ let divisor = if env::consts::OS == "macos" { 1024 } else { 1 };
+ let maxrss = (rusage.ru_maxrss + (divisor - 1)) / divisor;
+
+ let mut init_str = format!(
+ "user: {USER_SEC}.{USER_USEC:03} \
+ sys: {SYS_SEC}.{SYS_USEC:03} \
+ max rss (kb): {MAXRSS}",
+ USER_SEC = rusage.ru_utime.tv_sec,
+ USER_USEC = rusage.ru_utime.tv_usec,
+ SYS_SEC = rusage.ru_stime.tv_sec,
+ SYS_USEC = rusage.ru_stime.tv_usec,
+ MAXRSS = maxrss
+ );
+
+ // The remaining rusage stats vary in platform support. So we treat
+ // uniformly zero values in each category as "not worth printing", since it
+ // either means no events of that type occurred, or that the platform
+ // does not support it.
+
+ let minflt = rusage.ru_minflt;
+ let majflt = rusage.ru_majflt;
+ if minflt != 0 || majflt != 0 {
+ init_str.push_str(&format!(" page reclaims: {} page faults: {}", minflt, majflt));
+ }
+
+ let inblock = rusage.ru_inblock;
+ let oublock = rusage.ru_oublock;
+ if inblock != 0 || oublock != 0 {
+ init_str.push_str(&format!(" fs block inputs: {} fs block outputs: {}", inblock, oublock));
+ }
+
+ let nvcsw = rusage.ru_nvcsw;
+ let nivcsw = rusage.ru_nivcsw;
+ if nvcsw != 0 || nivcsw != 0 {
+ init_str.push_str(&format!(
+ " voluntary ctxt switches: {} involuntary ctxt switches: {}",
+ nvcsw, nivcsw
+ ));
+ }
+
+ return Some(init_str);
+}
diff --git a/src/bootstrap/bin/rustdoc.rs b/src/bootstrap/bin/rustdoc.rs
new file mode 100644
index 000000000..87c1d22e7
--- /dev/null
+++ b/src/bootstrap/bin/rustdoc.rs
@@ -0,0 +1,81 @@
+//! Shim which is passed to Cargo as "rustdoc" when running the bootstrap.
+//!
+//! See comments in `src/bootstrap/rustc.rs` for more information.
+
+use std::env;
+use std::ffi::OsString;
+use std::path::PathBuf;
+use std::process::Command;
+
+include!("../dylib_util.rs");
+
+fn main() {
+ let args = env::args_os().skip(1).collect::<Vec<_>>();
+ let rustdoc = env::var_os("RUSTDOC_REAL").expect("RUSTDOC_REAL was not set");
+ let libdir = env::var_os("RUSTDOC_LIBDIR").expect("RUSTDOC_LIBDIR was not set");
+ let sysroot = env::var_os("RUSTC_SYSROOT").expect("RUSTC_SYSROOT was not set");
+
+ // Detect whether or not we're a build script depending on whether --target
+ // is passed (a bit janky...)
+ let target = args.windows(2).find(|w| &*w[0] == "--target").and_then(|w| w[1].to_str());
+
+ use std::str::FromStr;
+
+ let verbose = match env::var("RUSTC_VERBOSE") {
+ Ok(s) => usize::from_str(&s).expect("RUSTC_VERBOSE should be an integer"),
+ Err(_) => 0,
+ };
+
+ let mut dylib_path = dylib_path();
+ dylib_path.insert(0, PathBuf::from(libdir.clone()));
+
+ let mut cmd = Command::new(rustdoc);
+
+ if target.is_some() {
+ // The stage0 compiler has a special sysroot distinct from what we
+ // actually downloaded, so we just always pass the `--sysroot` option,
+ // unless one is already set.
+ if !args.iter().any(|arg| arg == "--sysroot") {
+ cmd.arg("--sysroot").arg(&sysroot);
+ }
+ }
+
+ cmd.args(&args);
+ cmd.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap());
+
+ // Force all crates compiled by this compiler to (a) be unstable and (b)
+ // allow the `rustc_private` feature to link to other unstable crates
+ // also in the sysroot.
+ if env::var_os("RUSTC_FORCE_UNSTABLE").is_some() {
+ cmd.arg("-Z").arg("force-unstable-if-unmarked");
+ }
+ if let Some(linker) = env::var_os("RUSTDOC_LINKER") {
+ let mut arg = OsString::from("-Clinker=");
+ arg.push(&linker);
+ cmd.arg(arg);
+ }
+ if env::var_os("RUSTDOC_FUSE_LD_LLD").is_some() {
+ cmd.arg("-Clink-arg=-fuse-ld=lld");
+ if cfg!(windows) {
+ cmd.arg("-Clink-arg=-Wl,/threads:1");
+ } else {
+ cmd.arg("-Clink-arg=-Wl,--threads=1");
+ }
+ }
+
+ if verbose > 1 {
+ eprintln!(
+ "rustdoc command: {:?}={:?} {:?}",
+ dylib_path_var(),
+ env::join_paths(&dylib_path).unwrap(),
+ cmd,
+ );
+ eprintln!("sysroot: {:?}", sysroot);
+ eprintln!("libdir: {:?}", libdir);
+ }
+
+ std::process::exit(match cmd.status() {
+ Ok(s) => s.code().unwrap_or(1),
+ Err(e) => panic!("\n\nfailed to run {:?}: {}\n\n", cmd, e),
+ })
+}
diff --git a/src/bootstrap/bin/sccache-plus-cl.rs b/src/bootstrap/bin/sccache-plus-cl.rs
new file mode 100644
index 000000000..554c2dd4d
--- /dev/null
+++ b/src/bootstrap/bin/sccache-plus-cl.rs
@@ -0,0 +1,38 @@
+use std::env;
+use std::process::{self, Command};
+
+fn main() {
+ let target = env::var("SCCACHE_TARGET").unwrap();
+ // Locate the actual compiler that we're invoking
+ env::set_var("CC", env::var_os("SCCACHE_CC").unwrap());
+ env::set_var("CXX", env::var_os("SCCACHE_CXX").unwrap());
+ let mut cfg = cc::Build::new();
+ cfg.cargo_metadata(false)
+ .out_dir("/")
+ .target(&target)
+ .host(&target)
+ .opt_level(0)
+ .warnings(false)
+ .debug(false);
+ let compiler = cfg.get_compiler();
+
+ // Invoke sccache with said compiler
+ let sccache_path = env::var_os("SCCACHE_PATH").unwrap();
+ let mut cmd = Command::new(&sccache_path);
+ cmd.arg(compiler.path());
+ for &(ref k, ref v) in compiler.env() {
+ cmd.env(k, v);
+ }
+ for arg in env::args().skip(1) {
+ cmd.arg(arg);
+ }
+
+ if let Ok(s) = env::var("SCCACHE_EXTRA_ARGS") {
+ for s in s.split_whitespace() {
+ cmd.arg(s);
+ }
+ }
+
+ let status = cmd.status().expect("failed to spawn");
+ process::exit(status.code().unwrap_or(2))
+}
diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py
new file mode 100644
index 000000000..03eec02a8
--- /dev/null
+++ b/src/bootstrap/bootstrap.py
@@ -0,0 +1,945 @@
+from __future__ import absolute_import, division, print_function
+import argparse
+import contextlib
+import datetime
+import distutils.version
+import hashlib
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tarfile
+import tempfile
+
+from time import time, sleep
+
+def support_xz():
+ try:
+ with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+ temp_path = temp_file.name
+ with tarfile.open(temp_path, "w:xz"):
+ pass
+ return True
+ except tarfile.CompressionError:
+ return False
+
+def get(base, url, path, checksums, verbose=False):
+ with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+ temp_path = temp_file.name
+
+ try:
+ if url not in checksums:
+ raise RuntimeError(("src/stage0.json doesn't contain a checksum for {}. "
+ "Pre-built artifacts might not be available for this "
+ "target at this time, see https://doc.rust-lang.org/nightly"
+ "/rustc/platform-support.html for more information.")
+ .format(url))
+ sha256 = checksums[url]
+ if os.path.exists(path):
+ if verify(path, sha256, False):
+ if verbose:
+ print("using already-download file", path)
+ return
+ else:
+ if verbose:
+ print("ignoring already-download file",
+ path, "due to failed verification")
+ os.unlink(path)
+ download(temp_path, "{}/{}".format(base, url), True, verbose)
+ if not verify(temp_path, sha256, verbose):
+ raise RuntimeError("failed verification")
+ if verbose:
+ print("moving {} to {}".format(temp_path, path))
+ shutil.move(temp_path, path)
+ finally:
+ if os.path.isfile(temp_path):
+ if verbose:
+ print("removing", temp_path)
+ os.unlink(temp_path)
+
+
+def download(path, url, probably_big, verbose):
+ for _ in range(0, 4):
+ try:
+ _download(path, url, probably_big, verbose, True)
+ return
+ except RuntimeError:
+ print("\nspurious failure, trying again")
+ _download(path, url, probably_big, verbose, False)
+
+
+def _download(path, url, probably_big, verbose, exception):
+ # Try to use curl (potentially available on win32
+ # https://devblogs.microsoft.com/commandline/tar-and-curl-come-to-windows/)
+ # If an error occurs:
+ # - If we are on win32 fallback to powershell
+ # - Otherwise raise the error if appropriate
+ if probably_big or verbose:
+ print("downloading {}".format(url))
+
+ platform_is_win32 = sys.platform == 'win32'
+ try:
+ if probably_big or verbose:
+ option = "-#"
+ else:
+ option = "-s"
+ # If curl is not present on Win32, we shoud not sys.exit
+ # but raise `CalledProcessError` or `OSError` instead
+ require(["curl", "--version"], exception=platform_is_win32)
+ run(["curl", option,
+ "-L", # Follow redirect.
+ "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds
+ "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
+ "--retry", "3", "-Sf", "-o", path, url],
+ verbose=verbose,
+ exception=True, # Will raise RuntimeError on failure
+ )
+ except (subprocess.CalledProcessError, OSError, RuntimeError):
+ # see http://serverfault.com/questions/301128/how-to-download
+ if platform_is_win32:
+ run(["PowerShell.exe", "/nologo", "-Command",
+ "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
+ "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
+ verbose=verbose,
+ exception=exception)
+ # Check if the RuntimeError raised by run(curl) should be silenced
+ elif verbose or exception:
+ raise
+
+
+def verify(path, expected, verbose):
+ """Check if the sha256 sum of the given path is valid"""
+ if verbose:
+ print("verifying", path)
+ with open(path, "rb") as source:
+ found = hashlib.sha256(source.read()).hexdigest()
+ verified = found == expected
+ if not verified:
+ print("invalid checksum:\n"
+ " found: {}\n"
+ " expected: {}".format(found, expected))
+ return verified
+
+
+def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
+ """Unpack the given tarball file"""
+ print("extracting", tarball)
+ fname = os.path.basename(tarball).replace(tarball_suffix, "")
+ with contextlib.closing(tarfile.open(tarball)) as tar:
+ for member in tar.getnames():
+ if "/" not in member:
+ continue
+ name = member.replace(fname + "/", "", 1)
+ if match is not None and not name.startswith(match):
+ continue
+ name = name[len(match) + 1:]
+
+ dst_path = os.path.join(dst, name)
+ if verbose:
+ print(" extracting", member)
+ tar.extract(member, dst)
+ src_path = os.path.join(dst, member)
+ if os.path.isdir(src_path) and os.path.exists(dst_path):
+ continue
+ shutil.move(src_path, dst_path)
+ shutil.rmtree(os.path.join(dst, fname))
+
+
+def run(args, verbose=False, exception=False, is_bootstrap=False, **kwargs):
+ """Run a child program in a new process"""
+ if verbose:
+ print("running: " + ' '.join(args))
+ sys.stdout.flush()
+ # Ensure that the .exe is used on Windows just in case a Linux ELF has been
+ # compiled in the same directory.
+ if os.name == 'nt' and not args[0].endswith('.exe'):
+ args[0] += '.exe'
+ # Use Popen here instead of call() as it apparently allows powershell on
+ # Windows to not lock up waiting for input presumably.
+ ret = subprocess.Popen(args, **kwargs)
+ code = ret.wait()
+ if code != 0:
+ err = "failed to run: " + ' '.join(args)
+ if verbose or exception:
+ raise RuntimeError(err)
+ # For most failures, we definitely do want to print this error, or the user will have no
+ # idea what went wrong. But when we've successfully built bootstrap and it failed, it will
+ # have already printed an error above, so there's no need to print the exact command we're
+ # running.
+ if is_bootstrap:
+ sys.exit(1)
+ else:
+ sys.exit(err)
+
+
+def require(cmd, exit=True, exception=False):
+ '''Run a command, returning its output.
+ On error,
+ If `exception` is `True`, raise the error
+ Otherwise If `exit` is `True`, exit the process
+ Else return None.'''
+ try:
+ return subprocess.check_output(cmd).strip()
+ except (subprocess.CalledProcessError, OSError) as exc:
+ if exception:
+ raise
+ elif exit:
+ print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
+ print("Please make sure it's installed and in the path.")
+ sys.exit(1)
+ return None
+
+
+
+def format_build_time(duration):
+ """Return a nicer format for build time
+
+ >>> format_build_time('300')
+ '0:05:00'
+ """
+ return str(datetime.timedelta(seconds=int(duration)))
+
+
+def default_build_triple(verbose):
+ """Build triple as in LLVM"""
+ # If the user already has a host build triple with an existing `rustc`
+ # install, use their preference. This fixes most issues with Windows builds
+ # being detected as GNU instead of MSVC.
+ default_encoding = sys.getdefaultencoding()
+ try:
+ version = subprocess.check_output(["rustc", "--version", "--verbose"],
+ stderr=subprocess.DEVNULL)
+ version = version.decode(default_encoding)
+ host = next(x for x in version.split('\n') if x.startswith("host: "))
+ triple = host.split("host: ")[1]
+ if verbose:
+ print("detected default triple {} from pre-installed rustc".format(triple))
+ return triple
+ except Exception as e:
+ if verbose:
+ print("pre-installed rustc not detected: {}".format(e))
+ print("falling back to auto-detect")
+
+ required = sys.platform != 'win32'
+ ostype = require(["uname", "-s"], exit=required)
+ cputype = require(['uname', '-m'], exit=required)
+
+ # If we do not have `uname`, assume Windows.
+ if ostype is None or cputype is None:
+ return 'x86_64-pc-windows-msvc'
+
+ ostype = ostype.decode(default_encoding)
+ cputype = cputype.decode(default_encoding)
+
+ # The goal here is to come up with the same triple as LLVM would,
+ # at least for the subset of platforms we're willing to target.
+ ostype_mapper = {
+ 'Darwin': 'apple-darwin',
+ 'DragonFly': 'unknown-dragonfly',
+ 'FreeBSD': 'unknown-freebsd',
+ 'Haiku': 'unknown-haiku',
+ 'NetBSD': 'unknown-netbsd',
+ 'OpenBSD': 'unknown-openbsd'
+ }
+
+ # Consider the direct transformation first and then the special cases
+ if ostype in ostype_mapper:
+ ostype = ostype_mapper[ostype]
+ elif ostype == 'Linux':
+ os_from_sp = subprocess.check_output(
+ ['uname', '-o']).strip().decode(default_encoding)
+ if os_from_sp == 'Android':
+ ostype = 'linux-android'
+ else:
+ ostype = 'unknown-linux-gnu'
+ elif ostype == 'SunOS':
+ ostype = 'pc-solaris'
+ # On Solaris, uname -m will return a machine classification instead
+ # of a cpu type, so uname -p is recommended instead. However, the
+ # output from that option is too generic for our purposes (it will
+ # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
+ # must be used instead.
+ cputype = require(['isainfo', '-k']).decode(default_encoding)
+ # sparc cpus have sun as a target vendor
+ if 'sparc' in cputype:
+ ostype = 'sun-solaris'
+ elif ostype.startswith('MINGW'):
+ # msys' `uname` does not print gcc configuration, but prints msys
+ # configuration. so we cannot believe `uname -m`:
+ # msys1 is always i686 and msys2 is always x86_64.
+ # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
+ # MINGW64 on x86_64.
+ ostype = 'pc-windows-gnu'
+ cputype = 'i686'
+ if os.environ.get('MSYSTEM') == 'MINGW64':
+ cputype = 'x86_64'
+ elif ostype.startswith('MSYS'):
+ ostype = 'pc-windows-gnu'
+ elif ostype.startswith('CYGWIN_NT'):
+ cputype = 'i686'
+ if ostype.endswith('WOW64'):
+ cputype = 'x86_64'
+ ostype = 'pc-windows-gnu'
+ elif sys.platform == 'win32':
+ # Some Windows platforms might have a `uname` command that returns a
+ # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
+ # these cases, fall back to using sys.platform.
+ return 'x86_64-pc-windows-msvc'
+ else:
+ err = "unknown OS type: {}".format(ostype)
+ sys.exit(err)
+
+ if cputype in ['powerpc', 'riscv'] and ostype == 'unknown-freebsd':
+ cputype = subprocess.check_output(
+ ['uname', '-p']).strip().decode(default_encoding)
+ cputype_mapper = {
+ 'BePC': 'i686',
+ 'aarch64': 'aarch64',
+ 'amd64': 'x86_64',
+ 'arm64': 'aarch64',
+ 'i386': 'i686',
+ 'i486': 'i686',
+ 'i686': 'i686',
+ 'i786': 'i686',
+ 'm68k': 'm68k',
+ 'powerpc': 'powerpc',
+ 'powerpc64': 'powerpc64',
+ 'powerpc64le': 'powerpc64le',
+ 'ppc': 'powerpc',
+ 'ppc64': 'powerpc64',
+ 'ppc64le': 'powerpc64le',
+ 'riscv64': 'riscv64gc',
+ 's390x': 's390x',
+ 'x64': 'x86_64',
+ 'x86': 'i686',
+ 'x86-64': 'x86_64',
+ 'x86_64': 'x86_64'
+ }
+
+ # Consider the direct transformation first and then the special cases
+ if cputype in cputype_mapper:
+ cputype = cputype_mapper[cputype]
+ elif cputype in {'xscale', 'arm'}:
+ cputype = 'arm'
+ if ostype == 'linux-android':
+ ostype = 'linux-androideabi'
+ elif ostype == 'unknown-freebsd':
+ cputype = subprocess.check_output(
+ ['uname', '-p']).strip().decode(default_encoding)
+ ostype = 'unknown-freebsd'
+ elif cputype == 'armv6l':
+ cputype = 'arm'
+ if ostype == 'linux-android':
+ ostype = 'linux-androideabi'
+ else:
+ ostype += 'eabihf'
+ elif cputype in {'armv7l', 'armv8l'}:
+ cputype = 'armv7'
+ if ostype == 'linux-android':
+ ostype = 'linux-androideabi'
+ else:
+ ostype += 'eabihf'
+ elif cputype == 'mips':
+ if sys.byteorder == 'big':
+ cputype = 'mips'
+ elif sys.byteorder == 'little':
+ cputype = 'mipsel'
+ else:
+ raise ValueError("unknown byteorder: {}".format(sys.byteorder))
+ elif cputype == 'mips64':
+ if sys.byteorder == 'big':
+ cputype = 'mips64'
+ elif sys.byteorder == 'little':
+ cputype = 'mips64el'
+ else:
+ raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
+ # only the n64 ABI is supported, indicate it
+ ostype += 'abi64'
+ elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
+ pass
+ else:
+ err = "unknown cpu type: {}".format(cputype)
+ sys.exit(err)
+
+ return "{}-{}".format(cputype, ostype)
+
+
+@contextlib.contextmanager
+def output(filepath):
+ tmp = filepath + '.tmp'
+ with open(tmp, 'w') as f:
+ yield f
+ try:
+ if os.path.exists(filepath):
+ os.remove(filepath) # PermissionError/OSError on Win32 if in use
+ except OSError:
+ shutil.copy2(tmp, filepath)
+ os.remove(tmp)
+ return
+ os.rename(tmp, filepath)
+
+
+class Stage0Toolchain:
+ def __init__(self, stage0_payload):
+ self.date = stage0_payload["date"]
+ self.version = stage0_payload["version"]
+
+ def channel(self):
+ return self.version + "-" + self.date
+
+
+class RustBuild(object):
+ """Provide all the methods required to build Rust"""
+ def __init__(self):
+ self.checksums_sha256 = {}
+ self.stage0_compiler = None
+ self._download_url = ''
+ self.build = ''
+ self.build_dir = ''
+ self.clean = False
+ self.config_toml = ''
+ self.rust_root = ''
+ self.use_locked_deps = ''
+ self.use_vendored_sources = ''
+ self.verbose = False
+ self.git_version = None
+ self.nix_deps_dir = None
+
+ def download_toolchain(self):
+ """Fetch the build system for Rust, written in Rust
+
+ This method will build a cache directory, then it will fetch the
+ tarball which has the stage0 compiler used to then bootstrap the Rust
+ compiler itself.
+
+ Each downloaded tarball is extracted, after that, the script
+ will move all the content to the right place.
+ """
+ rustc_channel = self.stage0_compiler.version
+ bin_root = self.bin_root()
+
+ key = self.stage0_compiler.date
+ if self.rustc().startswith(bin_root) and \
+ (not os.path.exists(self.rustc()) or
+ self.program_out_of_date(self.rustc_stamp(), key)):
+ if os.path.exists(bin_root):
+ shutil.rmtree(bin_root)
+ tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
+ filename = "rust-std-{}-{}{}".format(
+ rustc_channel, self.build, tarball_suffix)
+ pattern = "rust-std-{}".format(self.build)
+ self._download_component_helper(filename, pattern, tarball_suffix)
+ filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
+ tarball_suffix)
+ self._download_component_helper(filename, "rustc", tarball_suffix)
+ filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
+ tarball_suffix)
+ self._download_component_helper(filename, "cargo", tarball_suffix)
+ self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
+
+ self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
+ self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
+ lib_dir = "{}/lib".format(bin_root)
+ for lib in os.listdir(lib_dir):
+ if lib.endswith(".so"):
+ self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
+ with output(self.rustc_stamp()) as rust_stamp:
+ rust_stamp.write(key)
+
+ def _download_component_helper(
+ self, filename, pattern, tarball_suffix,
+ ):
+ key = self.stage0_compiler.date
+ cache_dst = os.path.join(self.build_dir, "cache")
+ rustc_cache = os.path.join(cache_dst, key)
+ if not os.path.exists(rustc_cache):
+ os.makedirs(rustc_cache)
+
+ base = self._download_url
+ url = "dist/{}".format(key)
+ tarball = os.path.join(rustc_cache, filename)
+ if not os.path.exists(tarball):
+ get(
+ base,
+ "{}/{}".format(url, filename),
+ tarball,
+ self.checksums_sha256,
+ verbose=self.verbose,
+ )
+ unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
+
+ def fix_bin_or_dylib(self, fname):
+ """Modifies the interpreter section of 'fname' to fix the dynamic linker,
+ or the RPATH section, to fix the dynamic library search path
+
+ This method is only required on NixOS and uses the PatchELF utility to
+ change the interpreter/RPATH of ELF executables.
+
+ Please see https://nixos.org/patchelf.html for more information
+ """
+ default_encoding = sys.getdefaultencoding()
+ try:
+ ostype = subprocess.check_output(
+ ['uname', '-s']).strip().decode(default_encoding)
+ except subprocess.CalledProcessError:
+ return
+ except OSError as reason:
+ if getattr(reason, 'winerror', None) is not None:
+ return
+ raise reason
+
+ if ostype != "Linux":
+ return
+
+ # If the user has asked binaries to be patched for Nix, then
+ # don't check for NixOS or `/lib`, just continue to the patching.
+ if self.get_toml('patch-binaries-for-nix', 'build') != 'true':
+ # Use `/etc/os-release` instead of `/etc/NIXOS`.
+ # The latter one does not exist on NixOS when using tmpfs as root.
+ try:
+ with open("/etc/os-release", "r") as f:
+ if not any(l.strip() in ["ID=nixos", "ID='nixos'", 'ID="nixos"'] for l in f):
+ return
+ except FileNotFoundError:
+ return
+ if os.path.exists("/lib"):
+ return
+
+ # At this point we're pretty sure the user is running NixOS or
+ # using Nix
+ nix_os_msg = "info: you seem to be using Nix. Attempting to patch"
+ print(nix_os_msg, fname)
+
+ # Only build `.nix-deps` once.
+ nix_deps_dir = self.nix_deps_dir
+ if not nix_deps_dir:
+ # Run `nix-build` to "build" each dependency (which will likely reuse
+ # the existing `/nix/store` copy, or at most download a pre-built copy).
+ #
+ # Importantly, we create a gc-root called `.nix-deps` in the `build/`
+ # directory, but still reference the actual `/nix/store` path in the rpath
+ # as it makes it significantly more robust against changes to the location of
+ # the `.nix-deps` location.
+ #
+ # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
+ # zlib: Needed as a system dependency of `libLLVM-*.so`.
+ # patchelf: Needed for patching ELF binaries (see doc comment above).
+ nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps")
+ nix_expr = '''
+ with (import <nixpkgs> {});
+ symlinkJoin {
+ name = "rust-stage0-dependencies";
+ paths = [
+ zlib
+ patchelf
+ stdenv.cc.bintools
+ ];
+ }
+ '''
+ try:
+ subprocess.check_output([
+ "nix-build", "-E", nix_expr, "-o", nix_deps_dir,
+ ])
+ except subprocess.CalledProcessError as reason:
+ print("warning: failed to call nix-build:", reason)
+ return
+ self.nix_deps_dir = nix_deps_dir
+
+ patchelf = "{}/bin/patchelf".format(nix_deps_dir)
+ rpath_entries = [
+ # Relative default, all binary and dynamic libraries we ship
+ # appear to have this (even when `../lib` is redundant).
+ "$ORIGIN/../lib",
+ os.path.join(os.path.realpath(nix_deps_dir), "lib")
+ ]
+ patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
+ if not fname.endswith(".so"):
+ # Finally, set the corret .interp for binaries
+ with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker:
+ patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
+
+ try:
+ subprocess.check_output([patchelf] + patchelf_args + [fname])
+ except subprocess.CalledProcessError as reason:
+ print("warning: failed to call patchelf:", reason)
+ return
+
+ def rustc_stamp(self):
+ """Return the path for .rustc-stamp at the given stage
+
+ >>> rb = RustBuild()
+ >>> rb.build_dir = "build"
+ >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
+ True
+ """
+ return os.path.join(self.bin_root(), '.rustc-stamp')
+
+ def program_out_of_date(self, stamp_path, key):
+ """Check if the given program stamp is out of date"""
+ if not os.path.exists(stamp_path) or self.clean:
+ return True
+ with open(stamp_path, 'r') as stamp:
+ return key != stamp.read()
+
+ def bin_root(self):
+ """Return the binary root directory for the given stage
+
+ >>> rb = RustBuild()
+ >>> rb.build_dir = "build"
+ >>> rb.bin_root() == os.path.join("build", "stage0")
+ True
+
+ When the 'build' property is given should be a nested directory:
+
+ >>> rb.build = "devel"
+ >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
+ True
+ """
+ subdir = "stage0"
+ return os.path.join(self.build_dir, self.build, subdir)
+
+ def get_toml(self, key, section=None):
+ """Returns the value of the given key in config.toml, otherwise returns None
+
+ >>> rb = RustBuild()
+ >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
+ >>> rb.get_toml("key2")
+ 'value2'
+
+ If the key does not exist, the result is None:
+
+ >>> rb.get_toml("key3") is None
+ True
+
+ Optionally also matches the section the key appears in
+
+ >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
+ >>> rb.get_toml('key', 'a')
+ 'value1'
+ >>> rb.get_toml('key', 'b')
+ 'value2'
+ >>> rb.get_toml('key', 'c') is None
+ True
+
+ >>> rb.config_toml = 'key1 = true'
+ >>> rb.get_toml("key1")
+ 'true'
+ """
+
+ cur_section = None
+ for line in self.config_toml.splitlines():
+ section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
+ if section_match is not None:
+ cur_section = section_match.group(1)
+
+ match = re.match(r'^{}\s*=(.*)$'.format(key), line)
+ if match is not None:
+ value = match.group(1)
+ if section is None or section == cur_section:
+ return self.get_string(value) or value.strip()
+ return None
+
+ def cargo(self):
+ """Return config path for cargo"""
+ return self.program_config('cargo')
+
+ def rustc(self):
+ """Return config path for rustc"""
+ return self.program_config('rustc')
+
+ def program_config(self, program):
+ """Return config path for the given program at the given stage
+
+ >>> rb = RustBuild()
+ >>> rb.config_toml = 'rustc = "rustc"\\n'
+ >>> rb.program_config('rustc')
+ 'rustc'
+ >>> rb.config_toml = ''
+ >>> cargo_path = rb.program_config('cargo')
+ >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
+ ... "bin", "cargo")
+ True
+ """
+ config = self.get_toml(program)
+ if config:
+ return os.path.expanduser(config)
+ return os.path.join(self.bin_root(), "bin", "{}{}".format(
+ program, self.exe_suffix()))
+
+ @staticmethod
+ def get_string(line):
+ """Return the value between double quotes
+
+ >>> RustBuild.get_string(' "devel" ')
+ 'devel'
+ >>> RustBuild.get_string(" 'devel' ")
+ 'devel'
+ >>> RustBuild.get_string('devel') is None
+ True
+ >>> RustBuild.get_string(' "devel ')
+ ''
+ """
+ start = line.find('"')
+ if start != -1:
+ end = start + 1 + line[start + 1:].find('"')
+ return line[start + 1:end]
+ start = line.find('\'')
+ if start != -1:
+ end = start + 1 + line[start + 1:].find('\'')
+ return line[start + 1:end]
+ return None
+
+ @staticmethod
+ def exe_suffix():
+ """Return a suffix for executables"""
+ if sys.platform == 'win32':
+ return '.exe'
+ return ''
+
+ def bootstrap_binary(self):
+ """Return the path of the bootstrap binary
+
+ >>> rb = RustBuild()
+ >>> rb.build_dir = "build"
+ >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
+ ... "debug", "bootstrap")
+ True
+ """
+ return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
+
+ def build_bootstrap(self, color):
+ """Build bootstrap"""
+ print("Building rustbuild")
+ build_dir = os.path.join(self.build_dir, "bootstrap")
+ if self.clean and os.path.exists(build_dir):
+ shutil.rmtree(build_dir)
+ env = os.environ.copy()
+ # `CARGO_BUILD_TARGET` breaks bootstrap build.
+ # See also: <https://github.com/rust-lang/rust/issues/70208>.
+ if "CARGO_BUILD_TARGET" in env:
+ del env["CARGO_BUILD_TARGET"]
+ env["CARGO_TARGET_DIR"] = build_dir
+ env["RUSTC"] = self.rustc()
+ env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
+ (os.pathsep + env["LD_LIBRARY_PATH"]) \
+ if "LD_LIBRARY_PATH" in env else ""
+ env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
+ (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
+ if "DYLD_LIBRARY_PATH" in env else ""
+ env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
+ (os.pathsep + env["LIBRARY_PATH"]) \
+ if "LIBRARY_PATH" in env else ""
+
+ # preserve existing RUSTFLAGS
+ env.setdefault("RUSTFLAGS", "")
+ build_section = "target.{}".format(self.build)
+ target_features = []
+ if self.get_toml("crt-static", build_section) == "true":
+ target_features += ["+crt-static"]
+ elif self.get_toml("crt-static", build_section) == "false":
+ target_features += ["-crt-static"]
+ if target_features:
+ env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
+ target_linker = self.get_toml("linker", build_section)
+ if target_linker is not None:
+ env["RUSTFLAGS"] += " -C linker=" + target_linker
+ env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
+ env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
+ if self.get_toml("deny-warnings", "rust") != "false":
+ env["RUSTFLAGS"] += " -Dwarnings"
+
+ env["PATH"] = os.path.join(self.bin_root(), "bin") + \
+ os.pathsep + env["PATH"]
+ if not os.path.isfile(self.cargo()):
+ raise Exception("no cargo executable found at `{}`".format(
+ self.cargo()))
+ args = [self.cargo(), "build", "--manifest-path",
+ os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
+ for _ in range(0, self.verbose):
+ args.append("--verbose")
+ if self.use_locked_deps:
+ args.append("--locked")
+ if self.use_vendored_sources:
+ args.append("--frozen")
+ if self.get_toml("metrics", "build"):
+ args.append("--features")
+ args.append("build-metrics")
+ if color == "always":
+ args.append("--color=always")
+ elif color == "never":
+ args.append("--color=never")
+
+ run(args, env=env, verbose=self.verbose)
+
+ def build_triple(self):
+ """Build triple as in LLVM
+
+ Note that `default_build_triple` is moderately expensive,
+ so use `self.build` where possible.
+ """
+ config = self.get_toml('build')
+ if config:
+ return config
+ return default_build_triple(self.verbose)
+
+ def set_dist_environment(self, url):
+ """Set download URL for normal environment"""
+ if 'RUSTUP_DIST_SERVER' in os.environ:
+ self._download_url = os.environ['RUSTUP_DIST_SERVER']
+ else:
+ self._download_url = url
+
+ def check_vendored_status(self):
+ """Check that vendoring is configured properly"""
+ # keep this consistent with the equivalent check in rustbuild:
+ # https://github.com/rust-lang/rust/blob/a8a33cf27166d3eabaffc58ed3799e054af3b0c6/src/bootstrap/lib.rs#L399-L405
+ if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
+ if os.getuid() == 0:
+ self.use_vendored_sources = True
+ print('info: looks like you\'re trying to run this command as root')
+ print(' and so in order to preserve your $HOME this will now')
+ print(' use vendored sources by default.')
+
+ cargo_dir = os.path.join(self.rust_root, '.cargo')
+ if self.use_vendored_sources:
+ vendor_dir = os.path.join(self.rust_root, 'vendor')
+ if not os.path.exists(vendor_dir):
+ sync_dirs = "--sync ./src/tools/rust-analyzer/Cargo.toml " \
+ "--sync ./compiler/rustc_codegen_cranelift/Cargo.toml " \
+ "--sync ./src/bootstrap/Cargo.toml "
+ print('error: vendoring required, but vendor directory does not exist.')
+ print(' Run `cargo vendor {}` to initialize the '
+ 'vendor directory.'.format(sync_dirs))
+ print('Alternatively, use the pre-vendored `rustc-src` dist component.')
+ raise Exception("{} not found".format(vendor_dir))
+
+ if not os.path.exists(cargo_dir):
+ print('error: vendoring required, but .cargo/config does not exist.')
+ raise Exception("{} not found".format(cargo_dir))
+ else:
+ if os.path.exists(cargo_dir):
+ shutil.rmtree(cargo_dir)
+
+def bootstrap(help_triggered):
+ """Configure, fetch, build and run the initial bootstrap"""
+
+ # If the user is asking for help, let them know that the whole download-and-build
+ # process has to happen before anything is printed out.
+ if help_triggered:
+ print("info: Downloading and building bootstrap before processing --help")
+ print(" command. See src/bootstrap/README.md for help with common")
+ print(" commands.")
+
+ parser = argparse.ArgumentParser(description='Build rust')
+ parser.add_argument('--config')
+ parser.add_argument('--build-dir')
+ parser.add_argument('--build')
+ parser.add_argument('--color', choices=['always', 'never', 'auto'])
+ parser.add_argument('--clean', action='store_true')
+ parser.add_argument('-v', '--verbose', action='count', default=0)
+
+ args = [a for a in sys.argv if a != '-h' and a != '--help']
+ args, _ = parser.parse_known_args(args)
+
+ # Configure initial bootstrap
+ build = RustBuild()
+ build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
+ build.verbose = args.verbose
+ build.clean = args.clean
+
+ # Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`,
+ # then `config.toml` in the root directory.
+ toml_path = args.config or os.getenv('RUST_BOOTSTRAP_CONFIG')
+ using_default_path = toml_path is None
+ if using_default_path:
+ toml_path = 'config.toml'
+ if not os.path.exists(toml_path):
+ toml_path = os.path.join(build.rust_root, toml_path)
+
+ # Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path,
+ # but not if `config.toml` hasn't been created.
+ if not using_default_path or os.path.exists(toml_path):
+ with open(toml_path) as config:
+ build.config_toml = config.read()
+
+ profile = build.get_toml('profile')
+ if profile is not None:
+ include_file = 'config.{}.toml'.format(profile)
+ include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
+ include_path = os.path.join(include_dir, include_file)
+ # HACK: This works because `build.get_toml()` returns the first match it finds for a
+ # specific key, so appending our defaults at the end allows the user to override them
+ with open(include_path) as included_toml:
+ build.config_toml += os.linesep + included_toml.read()
+
+ config_verbose = build.get_toml('verbose', 'build')
+ if config_verbose is not None:
+ build.verbose = max(build.verbose, int(config_verbose))
+
+ build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
+
+ build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
+
+ build.check_vendored_status()
+
+ build_dir = args.build_dir or build.get_toml('build-dir', 'build') or 'build'
+ build.build_dir = os.path.abspath(build_dir)
+
+ with open(os.path.join(build.rust_root, "src", "stage0.json")) as f:
+ data = json.load(f)
+ build.checksums_sha256 = data["checksums_sha256"]
+ build.stage0_compiler = Stage0Toolchain(data["compiler"])
+
+ build.set_dist_environment(data["config"]["dist_server"])
+
+ build.build = args.build or build.build_triple()
+
+ if not os.path.exists(build.build_dir):
+ os.makedirs(build.build_dir)
+
+ # Fetch/build the bootstrap
+ build.download_toolchain()
+ sys.stdout.flush()
+ build.build_bootstrap(args.color)
+ sys.stdout.flush()
+
+ # Run the bootstrap
+ args = [build.bootstrap_binary()]
+ args.extend(sys.argv[1:])
+ env = os.environ.copy()
+ env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
+ env["BOOTSTRAP_PYTHON"] = sys.executable
+ run(args, env=env, verbose=build.verbose, is_bootstrap=True)
+
+
+def main():
+ """Entry point for the bootstrap process"""
+ start_time = time()
+
+ # x.py help <cmd> ...
+ if len(sys.argv) > 1 and sys.argv[1] == 'help':
+ sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
+
+ help_triggered = (
+ '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
+ try:
+ bootstrap(help_triggered)
+ if not help_triggered:
+ print("Build completed successfully in {}".format(
+ format_build_time(time() - start_time)))
+ except (SystemExit, KeyboardInterrupt) as error:
+ if hasattr(error, 'code') and isinstance(error.code, int):
+ exit_code = error.code
+ else:
+ exit_code = 1
+ print(error)
+ if not help_triggered:
+ print("Build completed unsuccessfully in {}".format(
+ format_build_time(time() - start_time)))
+ sys.exit(exit_code)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/src/bootstrap/bootstrap_test.py b/src/bootstrap/bootstrap_test.py
new file mode 100644
index 000000000..06ca3ce21
--- /dev/null
+++ b/src/bootstrap/bootstrap_test.py
@@ -0,0 +1,87 @@
+"""Bootstrap tests"""
+
+from __future__ import absolute_import, division, print_function
+import os
+import doctest
+import unittest
+import tempfile
+import hashlib
+import sys
+
+from shutil import rmtree
+
+import bootstrap
+
+
+class VerifyTestCase(unittest.TestCase):
+ """Test Case for verify"""
+ def setUp(self):
+ self.container = tempfile.mkdtemp()
+ self.src = os.path.join(self.container, "src.txt")
+ self.bad_src = os.path.join(self.container, "bad.txt")
+ content = "Hello world"
+
+ self.expected = hashlib.sha256(content.encode("utf-8")).hexdigest()
+
+ with open(self.src, "w") as src:
+ src.write(content)
+ with open(self.bad_src, "w") as bad:
+ bad.write("Hello!")
+
+ def tearDown(self):
+ rmtree(self.container)
+
+ def test_valid_file(self):
+ """Check if the sha256 sum of the given file is valid"""
+ self.assertTrue(bootstrap.verify(self.src, self.expected, False))
+
+ def test_invalid_file(self):
+ """Should verify that the file is invalid"""
+ self.assertFalse(bootstrap.verify(self.bad_src, self.expected, False))
+
+
+class ProgramOutOfDate(unittest.TestCase):
+ """Test if a program is out of date"""
+ def setUp(self):
+ self.container = tempfile.mkdtemp()
+ os.mkdir(os.path.join(self.container, "stage0"))
+ self.build = bootstrap.RustBuild()
+ self.build.date = "2017-06-15"
+ self.build.build_dir = self.container
+ self.rustc_stamp_path = os.path.join(self.container, "stage0",
+ ".rustc-stamp")
+ self.key = self.build.date + str(None)
+
+ def tearDown(self):
+ rmtree(self.container)
+
+ def test_stamp_path_does_not_exist(self):
+ """Return True when the stamp file does not exist"""
+ if os.path.exists(self.rustc_stamp_path):
+ os.unlink(self.rustc_stamp_path)
+ self.assertTrue(self.build.program_out_of_date(self.rustc_stamp_path, self.key))
+
+ def test_dates_are_different(self):
+ """Return True when the dates are different"""
+ with open(self.rustc_stamp_path, "w") as rustc_stamp:
+ rustc_stamp.write("2017-06-14None")
+ self.assertTrue(self.build.program_out_of_date(self.rustc_stamp_path, self.key))
+
+ def test_same_dates(self):
+ """Return False both dates match"""
+ with open(self.rustc_stamp_path, "w") as rustc_stamp:
+ rustc_stamp.write("2017-06-15None")
+ self.assertFalse(self.build.program_out_of_date(self.rustc_stamp_path, self.key))
+
+
+if __name__ == '__main__':
+ SUITE = unittest.TestSuite()
+ TEST_LOADER = unittest.TestLoader()
+ SUITE.addTest(doctest.DocTestSuite(bootstrap))
+ SUITE.addTests([
+ TEST_LOADER.loadTestsFromTestCase(VerifyTestCase),
+ TEST_LOADER.loadTestsFromTestCase(ProgramOutOfDate)])
+
+ RUNNER = unittest.TextTestRunner(stream=sys.stdout, verbosity=2)
+ result = RUNNER.run(SUITE)
+ sys.exit(0 if result.wasSuccessful() else 1)
diff --git a/src/bootstrap/build.rs b/src/bootstrap/build.rs
new file mode 100644
index 000000000..ab34d5c1e
--- /dev/null
+++ b/src/bootstrap/build.rs
@@ -0,0 +1,43 @@
+use env::consts::{EXE_EXTENSION, EXE_SUFFIX};
+use std::env;
+use std::ffi::OsString;
+use std::path::PathBuf;
+
+/// Given an executable called `name`, return the filename for the
+/// executable for a particular target.
+pub fn exe(name: &PathBuf) -> PathBuf {
+ if EXE_EXTENSION != "" && name.extension() != Some(EXE_EXTENSION.as_ref()) {
+ let mut name: OsString = name.clone().into();
+ name.push(EXE_SUFFIX);
+ name.into()
+ } else {
+ name.clone()
+ }
+}
+
+fn main() {
+ let host = env::var("HOST").unwrap();
+ println!("cargo:rerun-if-changed=build.rs");
+ println!("cargo:rerun-if-env-changed=RUSTC");
+ println!("cargo:rustc-env=BUILD_TRIPLE={}", host);
+
+ // This may not be a canonicalized path.
+ let mut rustc = PathBuf::from(env::var_os("RUSTC").unwrap());
+
+ if rustc.is_relative() {
+ println!("cargo:rerun-if-env-changed=PATH");
+ for dir in env::split_paths(&env::var_os("PATH").unwrap_or_default()) {
+ let absolute = dir.join(&exe(&rustc));
+ if absolute.exists() {
+ rustc = absolute;
+ break;
+ }
+ }
+ }
+ assert!(rustc.is_absolute());
+
+ // FIXME: if the path is not utf-8, this is going to break. Unfortunately
+ // Cargo doesn't have a way for us to specify non-utf-8 paths easily, so
+ // we'll need to invent some encoding scheme if this becomes a problem.
+ println!("cargo:rustc-env=RUSTC={}", rustc.to_str().unwrap());
+}
diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs
new file mode 100644
index 000000000..0ab4824ac
--- /dev/null
+++ b/src/bootstrap/builder.rs
@@ -0,0 +1,2312 @@
+use std::any::{type_name, Any};
+use std::cell::{Cell, RefCell};
+use std::collections::BTreeSet;
+use std::env;
+use std::ffi::{OsStr, OsString};
+use std::fmt::{Debug, Write};
+use std::fs::{self, File};
+use std::hash::Hash;
+use std::io::{BufRead, BufReader, ErrorKind};
+use std::ops::Deref;
+use std::path::{Component, Path, PathBuf};
+use std::process::{Command, Stdio};
+use std::time::{Duration, Instant};
+
+use crate::cache::{Cache, Interned, INTERNER};
+use crate::config::{SplitDebuginfo, TargetSelection};
+use crate::dist;
+use crate::doc;
+use crate::flags::{Color, Subcommand};
+use crate::install;
+use crate::native;
+use crate::run;
+use crate::test;
+use crate::tool::{self, SourceType};
+use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir, output, t};
+use crate::EXTRA_CHECK_CFGS;
+use crate::{check, Config};
+use crate::{compile, Crate};
+use crate::{Build, CLang, DocTests, GitRepo, Mode};
+
+pub use crate::Compiler;
+// FIXME: replace with std::lazy after it gets stabilized and reaches beta
+use once_cell::sync::{Lazy, OnceCell};
+use xz2::bufread::XzDecoder;
+
+pub struct Builder<'a> {
+ pub build: &'a Build,
+ pub top_stage: u32,
+ pub kind: Kind,
+ cache: Cache,
+ stack: RefCell<Vec<Box<dyn Any>>>,
+ time_spent_on_dependencies: Cell<Duration>,
+ pub paths: Vec<PathBuf>,
+}
+
+impl<'a> Deref for Builder<'a> {
+ type Target = Build;
+
+ fn deref(&self) -> &Self::Target {
+ self.build
+ }
+}
+
+pub trait Step: 'static + Clone + Debug + PartialEq + Eq + Hash {
+ /// `PathBuf` when directories are created or to return a `Compiler` once
+ /// it's been assembled.
+ type Output: Clone;
+
+ /// Whether this step is run by default as part of its respective phase.
+ /// `true` here can still be overwritten by `should_run` calling `default_condition`.
+ const DEFAULT: bool = false;
+
+ /// If true, then this rule should be skipped if --target was specified, but --host was not
+ const ONLY_HOSTS: bool = false;
+
+ /// Primary function to execute this rule. Can call `builder.ensure()`
+ /// with other steps to run those.
+ fn run(self, builder: &Builder<'_>) -> Self::Output;
+
+ /// When bootstrap is passed a set of paths, this controls whether this rule
+ /// will execute. However, it does not get called in a "default" context
+ /// when we are not passed any paths; in that case, `make_run` is called
+ /// directly.
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_>;
+
+ /// Builds up a "root" rule, either as a default rule or from a path passed
+ /// to us.
+ ///
+ /// When path is `None`, we are executing in a context where no paths were
+ /// passed. When `./x.py build` is run, for example, this rule could get
+ /// called if it is in the correct list below with a path of `None`.
+ fn make_run(_run: RunConfig<'_>) {
+ // It is reasonable to not have an implementation of make_run for rules
+ // who do not want to get called from the root context. This means that
+ // they are likely dependencies (e.g., sysroot creation) or similar, and
+ // as such calling them from ./x.py isn't logical.
+ unimplemented!()
+ }
+}
+
+pub struct RunConfig<'a> {
+ pub builder: &'a Builder<'a>,
+ pub target: TargetSelection,
+ pub paths: Vec<PathSet>,
+}
+
+impl RunConfig<'_> {
+ pub fn build_triple(&self) -> TargetSelection {
+ self.builder.build.build
+ }
+}
+
+struct StepDescription {
+ default: bool,
+ only_hosts: bool,
+ should_run: fn(ShouldRun<'_>) -> ShouldRun<'_>,
+ make_run: fn(RunConfig<'_>),
+ name: &'static str,
+ kind: Kind,
+}
+
+#[derive(Clone, PartialOrd, Ord, PartialEq, Eq)]
+pub struct TaskPath {
+ pub path: PathBuf,
+ pub kind: Option<Kind>,
+}
+
+impl TaskPath {
+ pub fn parse(path: impl Into<PathBuf>) -> TaskPath {
+ let mut kind = None;
+ let mut path = path.into();
+
+ let mut components = path.components();
+ if let Some(Component::Normal(os_str)) = components.next() {
+ if let Some(str) = os_str.to_str() {
+ if let Some((found_kind, found_prefix)) = str.split_once("::") {
+ if found_kind.is_empty() {
+ panic!("empty kind in task path {}", path.display());
+ }
+ kind = Kind::parse(found_kind);
+ assert!(kind.is_some());
+ path = Path::new(found_prefix).join(components.as_path());
+ }
+ }
+ }
+
+ TaskPath { path, kind }
+ }
+}
+
+impl Debug for TaskPath {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ if let Some(kind) = &self.kind {
+ write!(f, "{}::", kind.as_str())?;
+ }
+ write!(f, "{}", self.path.display())
+ }
+}
+
+/// Collection of paths used to match a task rule.
+#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
+pub enum PathSet {
+ /// A collection of individual paths or aliases.
+ ///
+ /// These are generally matched as a path suffix. For example, a
+ /// command-line value of `std` will match if `library/std` is in the
+ /// set.
+ ///
+ /// NOTE: the paths within a set should always be aliases of one another.
+ /// For example, `src/librustdoc` and `src/tools/rustdoc` should be in the same set,
+ /// but `library/core` and `library/std` generally should not, unless there's no way (for that Step)
+ /// to build them separately.
+ Set(BTreeSet<TaskPath>),
+ /// A "suite" of paths.
+ ///
+ /// These can match as a path suffix (like `Set`), or as a prefix. For
+ /// example, a command-line value of `src/test/ui/abi/variadic-ffi.rs`
+ /// will match `src/test/ui`. A command-line value of `ui` would also
+ /// match `src/test/ui`.
+ Suite(TaskPath),
+}
+
+impl PathSet {
+ fn empty() -> PathSet {
+ PathSet::Set(BTreeSet::new())
+ }
+
+ fn one<P: Into<PathBuf>>(path: P, kind: Kind) -> PathSet {
+ let mut set = BTreeSet::new();
+ set.insert(TaskPath { path: path.into(), kind: Some(kind) });
+ PathSet::Set(set)
+ }
+
+ fn has(&self, needle: &Path, module: Option<Kind>) -> bool {
+ match self {
+ PathSet::Set(set) => set.iter().any(|p| Self::check(p, needle, module)),
+ PathSet::Suite(suite) => Self::check(suite, needle, module),
+ }
+ }
+
+ // internal use only
+ fn check(p: &TaskPath, needle: &Path, module: Option<Kind>) -> bool {
+ if let (Some(p_kind), Some(kind)) = (&p.kind, module) {
+ p.path.ends_with(needle) && *p_kind == kind
+ } else {
+ p.path.ends_with(needle)
+ }
+ }
+
+ /// Return all `TaskPath`s in `Self` that contain any of the `needles`, removing the
+ /// matched needles.
+ ///
+ /// This is used for `StepDescription::krate`, which passes all matching crates at once to
+ /// `Step::make_run`, rather than calling it many times with a single crate.
+ /// See `tests.rs` for examples.
+ fn intersection_removing_matches(
+ &self,
+ needles: &mut Vec<&Path>,
+ module: Option<Kind>,
+ ) -> PathSet {
+ let mut check = |p| {
+ for (i, n) in needles.iter().enumerate() {
+ let matched = Self::check(p, n, module);
+ if matched {
+ needles.remove(i);
+ return true;
+ }
+ }
+ false
+ };
+ match self {
+ PathSet::Set(set) => PathSet::Set(set.iter().filter(|&p| check(p)).cloned().collect()),
+ PathSet::Suite(suite) => {
+ if check(suite) {
+ self.clone()
+ } else {
+ PathSet::empty()
+ }
+ }
+ }
+ }
+
+ /// A convenience wrapper for Steps which know they have no aliases and all their sets contain only a single path.
+ ///
+ /// This can be used with [`ShouldRun::krate`], [`ShouldRun::path`], or [`ShouldRun::alias`].
+ #[track_caller]
+ pub fn assert_single_path(&self) -> &TaskPath {
+ match self {
+ PathSet::Set(set) => {
+ assert_eq!(set.len(), 1, "called assert_single_path on multiple paths");
+ set.iter().next().unwrap()
+ }
+ PathSet::Suite(_) => unreachable!("called assert_single_path on a Suite path"),
+ }
+ }
+}
+
+impl StepDescription {
+ fn from<S: Step>(kind: Kind) -> StepDescription {
+ StepDescription {
+ default: S::DEFAULT,
+ only_hosts: S::ONLY_HOSTS,
+ should_run: S::should_run,
+ make_run: S::make_run,
+ name: std::any::type_name::<S>(),
+ kind,
+ }
+ }
+
+ fn maybe_run(&self, builder: &Builder<'_>, pathsets: Vec<PathSet>) {
+ if pathsets.iter().any(|set| self.is_excluded(builder, set)) {
+ return;
+ }
+
+ // Determine the targets participating in this rule.
+ let targets = if self.only_hosts { &builder.hosts } else { &builder.targets };
+
+ for target in targets {
+ let run = RunConfig { builder, paths: pathsets.clone(), target: *target };
+ (self.make_run)(run);
+ }
+ }
+
+ fn is_excluded(&self, builder: &Builder<'_>, pathset: &PathSet) -> bool {
+ if builder.config.exclude.iter().any(|e| pathset.has(&e.path, e.kind)) {
+ println!("Skipping {:?} because it is excluded", pathset);
+ return true;
+ }
+
+ if !builder.config.exclude.is_empty() {
+ builder.verbose(&format!(
+ "{:?} not skipped for {:?} -- not in {:?}",
+ pathset, self.name, builder.config.exclude
+ ));
+ }
+ false
+ }
+
+ fn run(v: &[StepDescription], builder: &Builder<'_>, paths: &[PathBuf]) {
+ let should_runs = v
+ .iter()
+ .map(|desc| (desc.should_run)(ShouldRun::new(builder, desc.kind)))
+ .collect::<Vec<_>>();
+
+ // sanity checks on rules
+ for (desc, should_run) in v.iter().zip(&should_runs) {
+ assert!(
+ !should_run.paths.is_empty(),
+ "{:?} should have at least one pathset",
+ desc.name
+ );
+ }
+
+ if paths.is_empty() || builder.config.include_default_paths {
+ for (desc, should_run) in v.iter().zip(&should_runs) {
+ if desc.default && should_run.is_really_default() {
+ desc.maybe_run(builder, should_run.paths.iter().cloned().collect());
+ }
+ }
+ }
+
+ // strip CurDir prefix if present
+ let mut paths: Vec<_> =
+ paths.into_iter().map(|p| p.strip_prefix(".").unwrap_or(p)).collect();
+
+ // Handle all test suite paths.
+ // (This is separate from the loop below to avoid having to handle multiple paths in `is_suite_path` somehow.)
+ paths.retain(|path| {
+ for (desc, should_run) in v.iter().zip(&should_runs) {
+ if let Some(suite) = should_run.is_suite_path(&path) {
+ desc.maybe_run(builder, vec![suite.clone()]);
+ return false;
+ }
+ }
+ true
+ });
+
+ if paths.is_empty() {
+ return;
+ }
+
+ // Handle all PathSets.
+ for (desc, should_run) in v.iter().zip(&should_runs) {
+ let pathsets = should_run.pathset_for_paths_removing_matches(&mut paths, desc.kind);
+ if !pathsets.is_empty() {
+ desc.maybe_run(builder, pathsets);
+ }
+ }
+
+ if !paths.is_empty() {
+ eprintln!("error: no `{}` rules matched {:?}", builder.kind.as_str(), paths,);
+ eprintln!(
+ "help: run `x.py {} --help --verbose` to show a list of available paths",
+ builder.kind.as_str()
+ );
+ eprintln!(
+ "note: if you are adding a new Step to bootstrap itself, make sure you register it with `describe!`"
+ );
+ crate::detail_exit(1);
+ }
+ }
+}
+
+enum ReallyDefault<'a> {
+ Bool(bool),
+ Lazy(Lazy<bool, Box<dyn Fn() -> bool + 'a>>),
+}
+
+pub struct ShouldRun<'a> {
+ pub builder: &'a Builder<'a>,
+ kind: Kind,
+
+ // use a BTreeSet to maintain sort order
+ paths: BTreeSet<PathSet>,
+
+ // If this is a default rule, this is an additional constraint placed on
+ // its run. Generally something like compiler docs being enabled.
+ is_really_default: ReallyDefault<'a>,
+}
+
+impl<'a> ShouldRun<'a> {
+ fn new(builder: &'a Builder<'_>, kind: Kind) -> ShouldRun<'a> {
+ ShouldRun {
+ builder,
+ kind,
+ paths: BTreeSet::new(),
+ is_really_default: ReallyDefault::Bool(true), // by default no additional conditions
+ }
+ }
+
+ pub fn default_condition(mut self, cond: bool) -> Self {
+ self.is_really_default = ReallyDefault::Bool(cond);
+ self
+ }
+
+ pub fn lazy_default_condition(mut self, lazy_cond: Box<dyn Fn() -> bool + 'a>) -> Self {
+ self.is_really_default = ReallyDefault::Lazy(Lazy::new(lazy_cond));
+ self
+ }
+
+ pub fn is_really_default(&self) -> bool {
+ match &self.is_really_default {
+ ReallyDefault::Bool(val) => *val,
+ ReallyDefault::Lazy(lazy) => *lazy.deref(),
+ }
+ }
+
+ /// Indicates it should run if the command-line selects the given crate or
+ /// any of its (local) dependencies.
+ ///
+ /// Compared to `krate`, this treats the dependencies as aliases for the
+ /// same job. Generally it is preferred to use `krate`, and treat each
+ /// individual path separately. For example `./x.py test src/liballoc`
+ /// (which uses `krate`) will test just `liballoc`. However, `./x.py check
+ /// src/liballoc` (which uses `all_krates`) will check all of `libtest`.
+ /// `all_krates` should probably be removed at some point.
+ pub fn all_krates(mut self, name: &str) -> Self {
+ let mut set = BTreeSet::new();
+ for krate in self.builder.in_tree_crates(name, None) {
+ let path = krate.local_path(self.builder);
+ set.insert(TaskPath { path, kind: Some(self.kind) });
+ }
+ self.paths.insert(PathSet::Set(set));
+ self
+ }
+
+ /// Indicates it should run if the command-line selects the given crate or
+ /// any of its (local) dependencies.
+ ///
+ /// `make_run` will be called a single time with all matching command-line paths.
+ pub fn crate_or_deps(self, name: &str) -> Self {
+ let crates = self.builder.in_tree_crates(name, None);
+ self.crates(crates)
+ }
+
+ /// Indicates it should run if the command-line selects any of the given crates.
+ ///
+ /// `make_run` will be called a single time with all matching command-line paths.
+ pub(crate) fn crates(mut self, crates: Vec<&Crate>) -> Self {
+ for krate in crates {
+ let path = krate.local_path(self.builder);
+ self.paths.insert(PathSet::one(path, self.kind));
+ }
+ self
+ }
+
+ // single alias, which does not correspond to any on-disk path
+ pub fn alias(mut self, alias: &str) -> Self {
+ assert!(
+ !self.builder.src.join(alias).exists(),
+ "use `builder.path()` for real paths: {}",
+ alias
+ );
+ self.paths.insert(PathSet::Set(
+ std::iter::once(TaskPath { path: alias.into(), kind: Some(self.kind) }).collect(),
+ ));
+ self
+ }
+
+ // single, non-aliased path
+ pub fn path(self, path: &str) -> Self {
+ self.paths(&[path])
+ }
+
+ // multiple aliases for the same job
+ pub fn paths(mut self, paths: &[&str]) -> Self {
+ self.paths.insert(PathSet::Set(
+ paths
+ .iter()
+ .map(|p| {
+ // FIXME(#96188): make sure this is actually a path.
+ // This currently breaks for paths within submodules.
+ //assert!(
+ // self.builder.src.join(p).exists(),
+ // "`should_run.paths` should correspond to real on-disk paths - use `alias` if there is no relevant path: {}",
+ // p
+ //);
+ TaskPath { path: p.into(), kind: Some(self.kind) }
+ })
+ .collect(),
+ ));
+ self
+ }
+
+ /// Handles individual files (not directories) within a test suite.
+ fn is_suite_path(&self, requested_path: &Path) -> Option<&PathSet> {
+ self.paths.iter().find(|pathset| match pathset {
+ PathSet::Suite(suite) => requested_path.starts_with(&suite.path),
+ PathSet::Set(_) => false,
+ })
+ }
+
+ pub fn suite_path(mut self, suite: &str) -> Self {
+ self.paths.insert(PathSet::Suite(TaskPath { path: suite.into(), kind: Some(self.kind) }));
+ self
+ }
+
+ // allows being more explicit about why should_run in Step returns the value passed to it
+ pub fn never(mut self) -> ShouldRun<'a> {
+ self.paths.insert(PathSet::empty());
+ self
+ }
+
+ /// Given a set of requested paths, return the subset which match the Step for this `ShouldRun`,
+ /// removing the matches from `paths`.
+ ///
+ /// NOTE: this returns multiple PathSets to allow for the possibility of multiple units of work
+ /// within the same step. For example, `test::Crate` allows testing multiple crates in the same
+ /// cargo invocation, which are put into separate sets because they aren't aliases.
+ ///
+ /// The reason we return PathSet instead of PathBuf is to allow for aliases that mean the same thing
+ /// (for now, just `all_krates` and `paths`, but we may want to add an `aliases` function in the future?)
+ fn pathset_for_paths_removing_matches(
+ &self,
+ paths: &mut Vec<&Path>,
+ kind: Kind,
+ ) -> Vec<PathSet> {
+ let mut sets = vec![];
+ for pathset in &self.paths {
+ let subset = pathset.intersection_removing_matches(paths, Some(kind));
+ if subset != PathSet::empty() {
+ sets.push(subset);
+ }
+ }
+ sets
+ }
+}
+
+#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug)]
+pub enum Kind {
+ Build,
+ Check,
+ Clippy,
+ Fix,
+ Format,
+ Test,
+ Bench,
+ Doc,
+ Clean,
+ Dist,
+ Install,
+ Run,
+ Setup,
+}
+
+impl Kind {
+ pub fn parse(string: &str) -> Option<Kind> {
+ // these strings, including the one-letter aliases, must match the x.py help text
+ Some(match string {
+ "build" | "b" => Kind::Build,
+ "check" | "c" => Kind::Check,
+ "clippy" => Kind::Clippy,
+ "fix" => Kind::Fix,
+ "fmt" => Kind::Format,
+ "test" | "t" => Kind::Test,
+ "bench" => Kind::Bench,
+ "doc" | "d" => Kind::Doc,
+ "clean" => Kind::Clean,
+ "dist" => Kind::Dist,
+ "install" => Kind::Install,
+ "run" | "r" => Kind::Run,
+ "setup" => Kind::Setup,
+ _ => return None,
+ })
+ }
+
+ pub fn as_str(&self) -> &'static str {
+ match self {
+ Kind::Build => "build",
+ Kind::Check => "check",
+ Kind::Clippy => "clippy",
+ Kind::Fix => "fix",
+ Kind::Format => "fmt",
+ Kind::Test => "test",
+ Kind::Bench => "bench",
+ Kind::Doc => "doc",
+ Kind::Clean => "clean",
+ Kind::Dist => "dist",
+ Kind::Install => "install",
+ Kind::Run => "run",
+ Kind::Setup => "setup",
+ }
+ }
+}
+
+impl<'a> Builder<'a> {
+ fn get_step_descriptions(kind: Kind) -> Vec<StepDescription> {
+ macro_rules! describe {
+ ($($rule:ty),+ $(,)?) => {{
+ vec![$(StepDescription::from::<$rule>(kind)),+]
+ }};
+ }
+ match kind {
+ Kind::Build => describe!(
+ compile::Std,
+ compile::Rustc,
+ compile::Assemble,
+ compile::CodegenBackend,
+ compile::StartupObjects,
+ tool::BuildManifest,
+ tool::Rustbook,
+ tool::ErrorIndex,
+ tool::UnstableBookGen,
+ tool::Tidy,
+ tool::Linkchecker,
+ tool::CargoTest,
+ tool::Compiletest,
+ tool::RemoteTestServer,
+ tool::RemoteTestClient,
+ tool::RustInstaller,
+ tool::Cargo,
+ tool::Rls,
+ tool::RustAnalyzer,
+ tool::RustAnalyzerProcMacroSrv,
+ tool::RustDemangler,
+ tool::Rustdoc,
+ tool::Clippy,
+ tool::CargoClippy,
+ native::Llvm,
+ native::Sanitizers,
+ tool::Rustfmt,
+ tool::Miri,
+ tool::CargoMiri,
+ native::Lld,
+ native::CrtBeginEnd
+ ),
+ Kind::Check | Kind::Clippy | Kind::Fix => describe!(
+ check::Std,
+ check::Rustc,
+ check::Rustdoc,
+ check::CodegenBackend,
+ check::Clippy,
+ check::Miri,
+ check::Rls,
+ check::RustAnalyzer,
+ check::Rustfmt,
+ check::Bootstrap
+ ),
+ Kind::Test => describe!(
+ crate::toolstate::ToolStateCheck,
+ test::ExpandYamlAnchors,
+ test::Tidy,
+ test::Ui,
+ test::RunPassValgrind,
+ test::MirOpt,
+ test::Codegen,
+ test::CodegenUnits,
+ test::Assembly,
+ test::Incremental,
+ test::Debuginfo,
+ test::UiFullDeps,
+ test::Rustdoc,
+ test::Pretty,
+ test::Crate,
+ test::CrateLibrustc,
+ test::CrateRustdoc,
+ test::CrateRustdocJsonTypes,
+ test::Linkcheck,
+ test::TierCheck,
+ test::Cargotest,
+ test::Cargo,
+ test::Rls,
+ test::RustAnalyzer,
+ test::ErrorIndex,
+ test::Distcheck,
+ test::RunMakeFullDeps,
+ test::Nomicon,
+ test::Reference,
+ test::RustdocBook,
+ test::RustByExample,
+ test::TheBook,
+ test::UnstableBook,
+ test::RustcBook,
+ test::LintDocs,
+ test::RustcGuide,
+ test::EmbeddedBook,
+ test::EditionGuide,
+ test::Rustfmt,
+ test::Miri,
+ test::Clippy,
+ test::RustDemangler,
+ test::CompiletestTest,
+ test::RustdocJSStd,
+ test::RustdocJSNotStd,
+ test::RustdocGUI,
+ test::RustdocTheme,
+ test::RustdocUi,
+ test::RustdocJson,
+ test::HtmlCheck,
+ // Run bootstrap close to the end as it's unlikely to fail
+ test::Bootstrap,
+ // Run run-make last, since these won't pass without make on Windows
+ test::RunMake,
+ ),
+ Kind::Bench => describe!(test::Crate, test::CrateLibrustc),
+ Kind::Doc => describe!(
+ doc::UnstableBook,
+ doc::UnstableBookGen,
+ doc::TheBook,
+ doc::Standalone,
+ doc::Std,
+ doc::Rustc,
+ doc::Rustdoc,
+ doc::Rustfmt,
+ doc::ErrorIndex,
+ doc::Nomicon,
+ doc::Reference,
+ doc::RustdocBook,
+ doc::RustByExample,
+ doc::RustcBook,
+ doc::CargoBook,
+ doc::Clippy,
+ doc::ClippyBook,
+ doc::Miri,
+ doc::EmbeddedBook,
+ doc::EditionGuide,
+ ),
+ Kind::Dist => describe!(
+ dist::Docs,
+ dist::RustcDocs,
+ dist::Mingw,
+ dist::Rustc,
+ dist::Std,
+ dist::RustcDev,
+ dist::Analysis,
+ dist::Src,
+ dist::Cargo,
+ dist::Rls,
+ dist::RustAnalyzer,
+ dist::Rustfmt,
+ dist::RustDemangler,
+ dist::Clippy,
+ dist::Miri,
+ dist::LlvmTools,
+ dist::RustDev,
+ dist::Extended,
+ // It seems that PlainSourceTarball somehow changes how some of the tools
+ // perceive their dependencies (see #93033) which would invalidate fingerprints
+ // and force us to rebuild tools after vendoring dependencies.
+ // To work around this, create the Tarball after building all the tools.
+ dist::PlainSourceTarball,
+ dist::BuildManifest,
+ dist::ReproducibleArtifacts,
+ ),
+ Kind::Install => describe!(
+ install::Docs,
+ install::Std,
+ install::Cargo,
+ install::Rls,
+ install::RustAnalyzer,
+ install::Rustfmt,
+ install::RustDemangler,
+ install::Clippy,
+ install::Miri,
+ install::Analysis,
+ install::Src,
+ install::Rustc
+ ),
+ Kind::Run => describe!(run::ExpandYamlAnchors, run::BuildManifest, run::BumpStage0),
+ // These commands either don't use paths, or they're special-cased in Build::build()
+ Kind::Clean | Kind::Format | Kind::Setup => vec![],
+ }
+ }
+
+ pub fn get_help(build: &Build, kind: Kind) -> Option<String> {
+ let step_descriptions = Builder::get_step_descriptions(kind);
+ if step_descriptions.is_empty() {
+ return None;
+ }
+
+ let builder = Self::new_internal(build, kind, vec![]);
+ let builder = &builder;
+ // The "build" kind here is just a placeholder, it will be replaced with something else in
+ // the following statement.
+ let mut should_run = ShouldRun::new(builder, Kind::Build);
+ for desc in step_descriptions {
+ should_run.kind = desc.kind;
+ should_run = (desc.should_run)(should_run);
+ }
+ let mut help = String::from("Available paths:\n");
+ let mut add_path = |path: &Path| {
+ t!(write!(help, " ./x.py {} {}\n", kind.as_str(), path.display()));
+ };
+ for pathset in should_run.paths {
+ match pathset {
+ PathSet::Set(set) => {
+ for path in set {
+ add_path(&path.path);
+ }
+ }
+ PathSet::Suite(path) => {
+ add_path(&path.path.join("..."));
+ }
+ }
+ }
+ Some(help)
+ }
+
+ fn new_internal(build: &Build, kind: Kind, paths: Vec<PathBuf>) -> Builder<'_> {
+ Builder {
+ build,
+ top_stage: build.config.stage,
+ kind,
+ cache: Cache::new(),
+ stack: RefCell::new(Vec::new()),
+ time_spent_on_dependencies: Cell::new(Duration::new(0, 0)),
+ paths,
+ }
+ }
+
+ pub fn new(build: &Build) -> Builder<'_> {
+ let (kind, paths) = match build.config.cmd {
+ Subcommand::Build { ref paths } => (Kind::Build, &paths[..]),
+ Subcommand::Check { ref paths } => (Kind::Check, &paths[..]),
+ Subcommand::Clippy { ref paths, .. } => (Kind::Clippy, &paths[..]),
+ Subcommand::Fix { ref paths } => (Kind::Fix, &paths[..]),
+ Subcommand::Doc { ref paths, .. } => (Kind::Doc, &paths[..]),
+ Subcommand::Test { ref paths, .. } => (Kind::Test, &paths[..]),
+ Subcommand::Bench { ref paths, .. } => (Kind::Bench, &paths[..]),
+ Subcommand::Dist { ref paths } => (Kind::Dist, &paths[..]),
+ Subcommand::Install { ref paths } => (Kind::Install, &paths[..]),
+ Subcommand::Run { ref paths } => (Kind::Run, &paths[..]),
+ Subcommand::Format { .. } => (Kind::Format, &[][..]),
+ Subcommand::Clean { .. } | Subcommand::Setup { .. } => {
+ panic!()
+ }
+ };
+
+ Self::new_internal(build, kind, paths.to_owned())
+ }
+
+ pub fn execute_cli(&self) {
+ self.run_step_descriptions(&Builder::get_step_descriptions(self.kind), &self.paths);
+ }
+
+ pub fn default_doc(&self, paths: &[PathBuf]) {
+ self.run_step_descriptions(&Builder::get_step_descriptions(Kind::Doc), paths);
+ }
+
+ /// NOTE: keep this in sync with `rustdoc::clean::utils::doc_rust_lang_org_channel`, or tests will fail on beta/stable.
+ pub fn doc_rust_lang_org_channel(&self) -> String {
+ let channel = match &*self.config.channel {
+ "stable" => &self.version,
+ "beta" => "beta",
+ "nightly" | "dev" => "nightly",
+ // custom build of rustdoc maybe? link to the latest stable docs just in case
+ _ => "stable",
+ };
+ "https://doc.rust-lang.org/".to_owned() + channel
+ }
+
+ fn run_step_descriptions(&self, v: &[StepDescription], paths: &[PathBuf]) {
+ StepDescription::run(v, self, paths);
+ }
+
+ /// Modifies the interpreter section of 'fname' to fix the dynamic linker,
+ /// or the RPATH section, to fix the dynamic library search path
+ ///
+ /// This is only required on NixOS and uses the PatchELF utility to
+ /// change the interpreter/RPATH of ELF executables.
+ ///
+ /// Please see https://nixos.org/patchelf.html for more information
+ pub(crate) fn fix_bin_or_dylib(&self, fname: &Path) {
+ // FIXME: cache NixOS detection?
+ match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() {
+ Err(_) => return,
+ Ok(output) if !output.status.success() => return,
+ Ok(output) => {
+ let mut s = output.stdout;
+ if s.last() == Some(&b'\n') {
+ s.pop();
+ }
+ if s != b"Linux" {
+ return;
+ }
+ }
+ }
+
+ // If the user has asked binaries to be patched for Nix, then
+ // don't check for NixOS or `/lib`, just continue to the patching.
+ // NOTE: this intentionally comes after the Linux check:
+ // - patchelf only works with ELF files, so no need to run it on Mac or Windows
+ // - On other Unix systems, there is no stable syscall interface, so Nix doesn't manage the global libc.
+ if !self.config.patch_binaries_for_nix {
+ // Use `/etc/os-release` instead of `/etc/NIXOS`.
+ // The latter one does not exist on NixOS when using tmpfs as root.
+ const NIX_IDS: &[&str] = &["ID=nixos", "ID='nixos'", "ID=\"nixos\""];
+ let os_release = match File::open("/etc/os-release") {
+ Err(e) if e.kind() == ErrorKind::NotFound => return,
+ Err(e) => panic!("failed to access /etc/os-release: {}", e),
+ Ok(f) => f,
+ };
+ if !BufReader::new(os_release).lines().any(|l| NIX_IDS.contains(&t!(l).trim())) {
+ return;
+ }
+ if Path::new("/lib").exists() {
+ return;
+ }
+ }
+
+ // At this point we're pretty sure the user is running NixOS or using Nix
+ println!("info: you seem to be using Nix. Attempting to patch {}", fname.display());
+
+ // Only build `.nix-deps` once.
+ static NIX_DEPS_DIR: OnceCell<PathBuf> = OnceCell::new();
+ let mut nix_build_succeeded = true;
+ let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| {
+ // Run `nix-build` to "build" each dependency (which will likely reuse
+ // the existing `/nix/store` copy, or at most download a pre-built copy).
+ //
+ // Importantly, we create a gc-root called `.nix-deps` in the `build/`
+ // directory, but still reference the actual `/nix/store` path in the rpath
+ // as it makes it significantly more robust against changes to the location of
+ // the `.nix-deps` location.
+ //
+ // bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
+ // zlib: Needed as a system dependency of `libLLVM-*.so`.
+ // patchelf: Needed for patching ELF binaries (see doc comment above).
+ let nix_deps_dir = self.out.join(".nix-deps");
+ const NIX_EXPR: &str = "
+ with (import <nixpkgs> {});
+ symlinkJoin {
+ name = \"rust-stage0-dependencies\";
+ paths = [
+ zlib
+ patchelf
+ stdenv.cc.bintools
+ ];
+ }
+ ";
+ nix_build_succeeded = self.try_run(Command::new("nix-build").args(&[
+ Path::new("-E"),
+ Path::new(NIX_EXPR),
+ Path::new("-o"),
+ &nix_deps_dir,
+ ]));
+ nix_deps_dir
+ });
+ if !nix_build_succeeded {
+ return;
+ }
+
+ let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf"));
+ let rpath_entries = {
+ // ORIGIN is a relative default, all binary and dynamic libraries we ship
+ // appear to have this (even when `../lib` is redundant).
+ // NOTE: there are only two paths here, delimited by a `:`
+ let mut entries = OsString::from("$ORIGIN/../lib:");
+ entries.push(t!(fs::canonicalize(nix_deps_dir)));
+ entries.push("/lib");
+ entries
+ };
+ patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]);
+ if !fname.extension().map_or(false, |ext| ext == "so") {
+ // Finally, set the corret .interp for binaries
+ let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker");
+ // FIXME: can we support utf8 here? `args` doesn't accept Vec<u8>, only OsString ...
+ let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path))));
+ patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]);
+ }
+
+ self.try_run(patchelf.arg(fname));
+ }
+
+ pub(crate) fn download_component(&self, url: &str, dest_path: &Path, help_on_error: &str) {
+ self.verbose(&format!("download {url}"));
+ // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/.
+ let tempfile = self.tempdir().join(dest_path.file_name().unwrap());
+ // While bootstrap itself only supports http and https downloads, downstream forks might
+ // need to download components from other protocols. The match allows them adding more
+ // protocols without worrying about merge conficts if we change the HTTP implementation.
+ match url.split_once("://").map(|(proto, _)| proto) {
+ Some("http") | Some("https") => {
+ self.download_http_with_retries(&tempfile, url, help_on_error)
+ }
+ Some(other) => panic!("unsupported protocol {other} in {url}"),
+ None => panic!("no protocol in {url}"),
+ }
+ t!(std::fs::rename(&tempfile, dest_path));
+ }
+
+ fn download_http_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) {
+ println!("downloading {}", url);
+ // Try curl. If that fails and we are on windows, fallback to PowerShell.
+ let mut curl = Command::new("curl");
+ curl.args(&[
+ "-#",
+ "-y",
+ "30",
+ "-Y",
+ "10", // timeout if speed is < 10 bytes/sec for > 30 seconds
+ "--connect-timeout",
+ "30", // timeout if cannot connect within 30 seconds
+ "--retry",
+ "3",
+ "-Sf",
+ "-o",
+ ]);
+ curl.arg(tempfile);
+ curl.arg(url);
+ if !self.check_run(&mut curl) {
+ if self.build.build.contains("windows-msvc") {
+ println!("Fallback to PowerShell");
+ for _ in 0..3 {
+ if self.try_run(Command::new("PowerShell.exe").args(&[
+ "/nologo",
+ "-Command",
+ "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
+ &format!(
+ "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')",
+ url, tempfile.to_str().expect("invalid UTF-8 not supported with powershell downloads"),
+ ),
+ ])) {
+ return;
+ }
+ println!("\nspurious failure, trying again");
+ }
+ }
+ if !help_on_error.is_empty() {
+ eprintln!("{}", help_on_error);
+ }
+ crate::detail_exit(1);
+ }
+ }
+
+ pub(crate) fn unpack(&self, tarball: &Path, dst: &Path, pattern: &str) {
+ println!("extracting {} to {}", tarball.display(), dst.display());
+ if !dst.exists() {
+ t!(fs::create_dir_all(dst));
+ }
+
+ // `tarball` ends with `.tar.xz`; strip that suffix
+ // example: `rust-dev-nightly-x86_64-unknown-linux-gnu`
+ let uncompressed_filename =
+ Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap();
+ let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap());
+
+ // decompress the file
+ let data = t!(File::open(tarball));
+ let decompressor = XzDecoder::new(BufReader::new(data));
+
+ let mut tar = tar::Archive::new(decompressor);
+ for member in t!(tar.entries()) {
+ let mut member = t!(member);
+ let original_path = t!(member.path()).into_owned();
+ // skip the top-level directory
+ if original_path == directory_prefix {
+ continue;
+ }
+ let mut short_path = t!(original_path.strip_prefix(directory_prefix));
+ if !short_path.starts_with(pattern) {
+ continue;
+ }
+ short_path = t!(short_path.strip_prefix(pattern));
+ let dst_path = dst.join(short_path);
+ self.verbose(&format!("extracting {} to {}", original_path.display(), dst.display()));
+ if !t!(member.unpack_in(dst)) {
+ panic!("path traversal attack ??");
+ }
+ let src_path = dst.join(original_path);
+ if src_path.is_dir() && dst_path.exists() {
+ continue;
+ }
+ t!(fs::rename(src_path, dst_path));
+ }
+ t!(fs::remove_dir_all(dst.join(directory_prefix)));
+ }
+
+ /// Returns whether the SHA256 checksum of `path` matches `expected`.
+ pub(crate) fn verify(&self, path: &Path, expected: &str) -> bool {
+ use sha2::Digest;
+
+ self.verbose(&format!("verifying {}", path.display()));
+ let mut hasher = sha2::Sha256::new();
+ // FIXME: this is ok for rustfmt (4.1 MB large at time of writing), but it seems memory-intensive for rustc and larger components.
+ // Consider using streaming IO instead?
+ let contents = if self.config.dry_run { vec![] } else { t!(fs::read(path)) };
+ hasher.update(&contents);
+ let found = hex::encode(hasher.finalize().as_slice());
+ let verified = found == expected;
+ if !verified && !self.config.dry_run {
+ println!(
+ "invalid checksum: \n\
+ found: {found}\n\
+ expected: {expected}",
+ );
+ }
+ return verified;
+ }
+
+ /// Obtain a compiler at a given stage and for a given host. Explicitly does
+ /// not take `Compiler` since all `Compiler` instances are meant to be
+ /// obtained through this function, since it ensures that they are valid
+ /// (i.e., built and assembled).
+ pub fn compiler(&self, stage: u32, host: TargetSelection) -> Compiler {
+ self.ensure(compile::Assemble { target_compiler: Compiler { stage, host } })
+ }
+
+ /// Similar to `compiler`, except handles the full-bootstrap option to
+ /// silently use the stage1 compiler instead of a stage2 compiler if one is
+ /// requested.
+ ///
+ /// Note that this does *not* have the side effect of creating
+ /// `compiler(stage, host)`, unlike `compiler` above which does have such
+ /// a side effect. The returned compiler here can only be used to compile
+ /// new artifacts, it can't be used to rely on the presence of a particular
+ /// sysroot.
+ ///
+ /// See `force_use_stage1` for documentation on what each argument is.
+ pub fn compiler_for(
+ &self,
+ stage: u32,
+ host: TargetSelection,
+ target: TargetSelection,
+ ) -> Compiler {
+ if self.build.force_use_stage1(Compiler { stage, host }, target) {
+ self.compiler(1, self.config.build)
+ } else {
+ self.compiler(stage, host)
+ }
+ }
+
+ pub fn sysroot(&self, compiler: Compiler) -> Interned<PathBuf> {
+ self.ensure(compile::Sysroot { compiler })
+ }
+
+ /// Returns the libdir where the standard library and other artifacts are
+ /// found for a compiler's sysroot.
+ pub fn sysroot_libdir(&self, compiler: Compiler, target: TargetSelection) -> Interned<PathBuf> {
+ #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+ struct Libdir {
+ compiler: Compiler,
+ target: TargetSelection,
+ }
+ impl Step for Libdir {
+ type Output = Interned<PathBuf>;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.never()
+ }
+
+ fn run(self, builder: &Builder<'_>) -> Interned<PathBuf> {
+ let lib = builder.sysroot_libdir_relative(self.compiler);
+ let sysroot = builder
+ .sysroot(self.compiler)
+ .join(lib)
+ .join("rustlib")
+ .join(self.target.triple)
+ .join("lib");
+ // Avoid deleting the rustlib/ directory we just copied
+ // (in `impl Step for Sysroot`).
+ if !builder.download_rustc() {
+ let _ = fs::remove_dir_all(&sysroot);
+ t!(fs::create_dir_all(&sysroot));
+ }
+ INTERNER.intern_path(sysroot)
+ }
+ }
+ self.ensure(Libdir { compiler, target })
+ }
+
+ pub fn sysroot_codegen_backends(&self, compiler: Compiler) -> PathBuf {
+ self.sysroot_libdir(compiler, compiler.host).with_file_name("codegen-backends")
+ }
+
+ /// Returns the compiler's libdir where it stores the dynamic libraries that
+ /// it itself links against.
+ ///
+ /// For example this returns `<sysroot>/lib` on Unix and `<sysroot>/bin` on
+ /// Windows.
+ pub fn rustc_libdir(&self, compiler: Compiler) -> PathBuf {
+ if compiler.is_snapshot(self) {
+ self.rustc_snapshot_libdir()
+ } else {
+ match self.config.libdir_relative() {
+ Some(relative_libdir) if compiler.stage >= 1 => {
+ self.sysroot(compiler).join(relative_libdir)
+ }
+ _ => self.sysroot(compiler).join(libdir(compiler.host)),
+ }
+ }
+ }
+
+ /// Returns the compiler's relative libdir where it stores the dynamic libraries that
+ /// it itself links against.
+ ///
+ /// For example this returns `lib` on Unix and `bin` on
+ /// Windows.
+ pub fn libdir_relative(&self, compiler: Compiler) -> &Path {
+ if compiler.is_snapshot(self) {
+ libdir(self.config.build).as_ref()
+ } else {
+ match self.config.libdir_relative() {
+ Some(relative_libdir) if compiler.stage >= 1 => relative_libdir,
+ _ => libdir(compiler.host).as_ref(),
+ }
+ }
+ }
+
+ /// Returns the compiler's relative libdir where the standard library and other artifacts are
+ /// found for a compiler's sysroot.
+ ///
+ /// For example this returns `lib` on Unix and Windows.
+ pub fn sysroot_libdir_relative(&self, compiler: Compiler) -> &Path {
+ match self.config.libdir_relative() {
+ Some(relative_libdir) if compiler.stage >= 1 => relative_libdir,
+ _ if compiler.stage == 0 => &self.build.initial_libdir,
+ _ => Path::new("lib"),
+ }
+ }
+
+ pub fn rustc_lib_paths(&self, compiler: Compiler) -> Vec<PathBuf> {
+ let mut dylib_dirs = vec![self.rustc_libdir(compiler)];
+
+ // Ensure that the downloaded LLVM libraries can be found.
+ if self.config.llvm_from_ci {
+ let ci_llvm_lib = self.out.join(&*compiler.host.triple).join("ci-llvm").join("lib");
+ dylib_dirs.push(ci_llvm_lib);
+ }
+
+ dylib_dirs
+ }
+
+ /// Adds the compiler's directory of dynamic libraries to `cmd`'s dynamic
+ /// library lookup path.
+ pub fn add_rustc_lib_path(&self, compiler: Compiler, cmd: &mut Command) {
+ // Windows doesn't need dylib path munging because the dlls for the
+ // compiler live next to the compiler and the system will find them
+ // automatically.
+ if cfg!(windows) {
+ return;
+ }
+
+ add_dylib_path(self.rustc_lib_paths(compiler), cmd);
+ }
+
+ /// Gets a path to the compiler specified.
+ pub fn rustc(&self, compiler: Compiler) -> PathBuf {
+ if compiler.is_snapshot(self) {
+ self.initial_rustc.clone()
+ } else {
+ self.sysroot(compiler).join("bin").join(exe("rustc", compiler.host))
+ }
+ }
+
+ /// Gets the paths to all of the compiler's codegen backends.
+ fn codegen_backends(&self, compiler: Compiler) -> impl Iterator<Item = PathBuf> {
+ fs::read_dir(self.sysroot_codegen_backends(compiler))
+ .into_iter()
+ .flatten()
+ .filter_map(Result::ok)
+ .map(|entry| entry.path())
+ }
+
+ pub fn rustdoc(&self, compiler: Compiler) -> PathBuf {
+ self.ensure(tool::Rustdoc { compiler })
+ }
+
+ pub fn rustdoc_cmd(&self, compiler: Compiler) -> Command {
+ let mut cmd = Command::new(&self.bootstrap_out.join("rustdoc"));
+ cmd.env("RUSTC_STAGE", compiler.stage.to_string())
+ .env("RUSTC_SYSROOT", self.sysroot(compiler))
+ // Note that this is *not* the sysroot_libdir because rustdoc must be linked
+ // equivalently to rustc.
+ .env("RUSTDOC_LIBDIR", self.rustc_libdir(compiler))
+ .env("CFG_RELEASE_CHANNEL", &self.config.channel)
+ .env("RUSTDOC_REAL", self.rustdoc(compiler))
+ .env("RUSTC_BOOTSTRAP", "1");
+
+ cmd.arg("-Wrustdoc::invalid_codeblock_attributes");
+
+ if self.config.deny_warnings {
+ cmd.arg("-Dwarnings");
+ }
+ cmd.arg("-Znormalize-docs");
+
+ // Remove make-related flags that can cause jobserver problems.
+ cmd.env_remove("MAKEFLAGS");
+ cmd.env_remove("MFLAGS");
+
+ if let Some(linker) = self.linker(compiler.host) {
+ cmd.env("RUSTDOC_LINKER", linker);
+ }
+ if self.is_fuse_ld_lld(compiler.host) {
+ cmd.env("RUSTDOC_FUSE_LD_LLD", "1");
+ }
+ cmd
+ }
+
+ /// Return the path to `llvm-config` for the target, if it exists.
+ ///
+ /// Note that this returns `None` if LLVM is disabled, or if we're in a
+ /// check build or dry-run, where there's no need to build all of LLVM.
+ fn llvm_config(&self, target: TargetSelection) -> Option<PathBuf> {
+ if self.config.llvm_enabled() && self.kind != Kind::Check && !self.config.dry_run {
+ let llvm_config = self.ensure(native::Llvm { target });
+ if llvm_config.is_file() {
+ return Some(llvm_config);
+ }
+ }
+ None
+ }
+
+ /// Convenience wrapper to allow `builder.llvm_link_shared()` instead of `builder.config.llvm_link_shared(&builder)`.
+ pub(crate) fn llvm_link_shared(&self) -> bool {
+ Config::llvm_link_shared(self)
+ }
+
+ pub(crate) fn download_rustc(&self) -> bool {
+ Config::download_rustc(self)
+ }
+
+ pub(crate) fn initial_rustfmt(&self) -> Option<PathBuf> {
+ Config::initial_rustfmt(self)
+ }
+
+ /// Prepares an invocation of `cargo` to be run.
+ ///
+ /// This will create a `Command` that represents a pending execution of
+ /// Cargo. This cargo will be configured to use `compiler` as the actual
+ /// rustc compiler, its output will be scoped by `mode`'s output directory,
+ /// it will pass the `--target` flag for the specified `target`, and will be
+ /// executing the Cargo command `cmd`.
+ pub fn cargo(
+ &self,
+ compiler: Compiler,
+ mode: Mode,
+ source_type: SourceType,
+ target: TargetSelection,
+ cmd: &str,
+ ) -> Cargo {
+ let mut cargo = Command::new(&self.initial_cargo);
+ let out_dir = self.stage_out(compiler, mode);
+
+ // Codegen backends are not yet tracked by -Zbinary-dep-depinfo,
+ // so we need to explicitly clear out if they've been updated.
+ for backend in self.codegen_backends(compiler) {
+ self.clear_if_dirty(&out_dir, &backend);
+ }
+
+ if cmd == "doc" || cmd == "rustdoc" {
+ let my_out = match mode {
+ // This is the intended out directory for compiler documentation.
+ Mode::Rustc | Mode::ToolRustc => self.compiler_doc_out(target),
+ Mode::Std => out_dir.join(target.triple).join("doc"),
+ _ => panic!("doc mode {:?} not expected", mode),
+ };
+ let rustdoc = self.rustdoc(compiler);
+ self.clear_if_dirty(&my_out, &rustdoc);
+ }
+
+ cargo.env("CARGO_TARGET_DIR", &out_dir).arg(cmd);
+
+ let profile_var = |name: &str| {
+ let profile = if self.config.rust_optimize { "RELEASE" } else { "DEV" };
+ format!("CARGO_PROFILE_{}_{}", profile, name)
+ };
+
+ // See comment in rustc_llvm/build.rs for why this is necessary, largely llvm-config
+ // needs to not accidentally link to libLLVM in stage0/lib.
+ cargo.env("REAL_LIBRARY_PATH_VAR", &util::dylib_path_var());
+ if let Some(e) = env::var_os(util::dylib_path_var()) {
+ cargo.env("REAL_LIBRARY_PATH", e);
+ }
+
+ // Found with `rg "init_env_logger\("`. If anyone uses `init_env_logger`
+ // from out of tree it shouldn't matter, since x.py is only used for
+ // building in-tree.
+ let color_logs = ["RUSTDOC_LOG_COLOR", "RUSTC_LOG_COLOR", "RUST_LOG_COLOR"];
+ match self.build.config.color {
+ Color::Always => {
+ cargo.arg("--color=always");
+ for log in &color_logs {
+ cargo.env(log, "always");
+ }
+ }
+ Color::Never => {
+ cargo.arg("--color=never");
+ for log in &color_logs {
+ cargo.env(log, "never");
+ }
+ }
+ Color::Auto => {} // nothing to do
+ }
+
+ if cmd != "install" {
+ cargo.arg("--target").arg(target.rustc_target_arg());
+ } else {
+ assert_eq!(target, compiler.host);
+ }
+
+ // Set a flag for `check`/`clippy`/`fix`, so that certain build
+ // scripts can do less work (i.e. not building/requiring LLVM).
+ if cmd == "check" || cmd == "clippy" || cmd == "fix" {
+ // If we've not yet built LLVM, or it's stale, then bust
+ // the rustc_llvm cache. That will always work, even though it
+ // may mean that on the next non-check build we'll need to rebuild
+ // rustc_llvm. But if LLVM is stale, that'll be a tiny amount
+ // of work comparatively, and we'd likely need to rebuild it anyway,
+ // so that's okay.
+ if crate::native::prebuilt_llvm_config(self, target).is_err() {
+ cargo.env("RUST_CHECK", "1");
+ }
+ }
+
+ let stage = if compiler.stage == 0 && self.local_rebuild {
+ // Assume the local-rebuild rustc already has stage1 features.
+ 1
+ } else {
+ compiler.stage
+ };
+
+ let mut rustflags = Rustflags::new(target);
+ if stage != 0 {
+ if let Ok(s) = env::var("CARGOFLAGS_NOT_BOOTSTRAP") {
+ cargo.args(s.split_whitespace());
+ }
+ rustflags.env("RUSTFLAGS_NOT_BOOTSTRAP");
+ } else {
+ if let Ok(s) = env::var("CARGOFLAGS_BOOTSTRAP") {
+ cargo.args(s.split_whitespace());
+ }
+ rustflags.env("RUSTFLAGS_BOOTSTRAP");
+ if cmd == "clippy" {
+ // clippy overwrites sysroot if we pass it to cargo.
+ // Pass it directly to clippy instead.
+ // NOTE: this can't be fixed in clippy because we explicitly don't set `RUSTC`,
+ // so it has no way of knowing the sysroot.
+ rustflags.arg("--sysroot");
+ rustflags.arg(
+ self.sysroot(compiler)
+ .as_os_str()
+ .to_str()
+ .expect("sysroot must be valid UTF-8"),
+ );
+ // Only run clippy on a very limited subset of crates (in particular, not build scripts).
+ cargo.arg("-Zunstable-options");
+ // Explicitly does *not* set `--cfg=bootstrap`, since we're using a nightly clippy.
+ let host_version = Command::new("rustc").arg("--version").output().map_err(|_| ());
+ let output = host_version.and_then(|output| {
+ if output.status.success() {
+ Ok(output)
+ } else {
+ Err(())
+ }
+ }).unwrap_or_else(|_| {
+ eprintln!(
+ "error: `x.py clippy` requires a host `rustc` toolchain with the `clippy` component"
+ );
+ eprintln!("help: try `rustup component add clippy`");
+ crate::detail_exit(1);
+ });
+ if !t!(std::str::from_utf8(&output.stdout)).contains("nightly") {
+ rustflags.arg("--cfg=bootstrap");
+ }
+ } else {
+ rustflags.arg("--cfg=bootstrap");
+ }
+ }
+
+ let use_new_symbol_mangling = match self.config.rust_new_symbol_mangling {
+ Some(setting) => {
+ // If an explicit setting is given, use that
+ setting
+ }
+ None => {
+ if mode == Mode::Std {
+ // The standard library defaults to the legacy scheme
+ false
+ } else {
+ // The compiler and tools default to the new scheme
+ true
+ }
+ }
+ };
+
+ if use_new_symbol_mangling {
+ rustflags.arg("-Csymbol-mangling-version=v0");
+ } else {
+ rustflags.arg("-Csymbol-mangling-version=legacy");
+ rustflags.arg("-Zunstable-options");
+ }
+
+ // Enable cfg checking of cargo features for everything but std and also enable cfg
+ // checking of names and values.
+ //
+ // Note: `std`, `alloc` and `core` imports some dependencies by #[path] (like
+ // backtrace, core_simd, std_float, ...), those dependencies have their own
+ // features but cargo isn't involved in the #[path] process and so cannot pass the
+ // complete list of features, so for that reason we don't enable checking of
+ // features for std crates.
+ cargo.arg(if mode != Mode::Std {
+ "-Zcheck-cfg=names,values,output,features"
+ } else {
+ "-Zcheck-cfg=names,values,output"
+ });
+
+ // Add extra cfg not defined in/by rustc
+ //
+ // Note: Altrough it would seems that "-Zunstable-options" to `rustflags` is useless as
+ // cargo would implicitly add it, it was discover that sometimes bootstrap only use
+ // `rustflags` without `cargo` making it required.
+ rustflags.arg("-Zunstable-options");
+ for (restricted_mode, name, values) in EXTRA_CHECK_CFGS {
+ if *restricted_mode == None || *restricted_mode == Some(mode) {
+ // Creating a string of the values by concatenating each value:
+ // ',"tvos","watchos"' or '' (nothing) when there are no values
+ let values = match values {
+ Some(values) => values
+ .iter()
+ .map(|val| [",", "\"", val, "\""])
+ .flatten()
+ .collect::<String>(),
+ None => String::new(),
+ };
+ rustflags.arg(&format!("--check-cfg=values({name}{values})"));
+ }
+ }
+
+ // FIXME: It might be better to use the same value for both `RUSTFLAGS` and `RUSTDOCFLAGS`,
+ // but this breaks CI. At the very least, stage0 `rustdoc` needs `--cfg bootstrap`. See
+ // #71458.
+ let mut rustdocflags = rustflags.clone();
+ rustdocflags.propagate_cargo_env("RUSTDOCFLAGS");
+ if stage == 0 {
+ rustdocflags.env("RUSTDOCFLAGS_BOOTSTRAP");
+ } else {
+ rustdocflags.env("RUSTDOCFLAGS_NOT_BOOTSTRAP");
+ }
+
+ if let Ok(s) = env::var("CARGOFLAGS") {
+ cargo.args(s.split_whitespace());
+ }
+
+ match mode {
+ Mode::Std | Mode::ToolBootstrap | Mode::ToolStd => {}
+ Mode::Rustc | Mode::Codegen | Mode::ToolRustc => {
+ // Build proc macros both for the host and the target
+ if target != compiler.host && cmd != "check" {
+ cargo.arg("-Zdual-proc-macros");
+ rustflags.arg("-Zdual-proc-macros");
+ }
+ }
+ }
+
+ // This tells Cargo (and in turn, rustc) to output more complete
+ // dependency information. Most importantly for rustbuild, this
+ // includes sysroot artifacts, like libstd, which means that we don't
+ // need to track those in rustbuild (an error prone process!). This
+ // feature is currently unstable as there may be some bugs and such, but
+ // it represents a big improvement in rustbuild's reliability on
+ // rebuilds, so we're using it here.
+ //
+ // For some additional context, see #63470 (the PR originally adding
+ // this), as well as #63012 which is the tracking issue for this
+ // feature on the rustc side.
+ cargo.arg("-Zbinary-dep-depinfo");
+ match mode {
+ Mode::ToolBootstrap => {
+ // Restrict the allowed features to those passed by rustbuild, so we don't depend on nightly accidentally.
+ // HACK: because anyhow does feature detection in build.rs, we need to allow the backtrace feature too.
+ rustflags.arg("-Zallow-features=binary-dep-depinfo,backtrace");
+ }
+ Mode::ToolStd => {
+ // Right now this is just compiletest and a few other tools that build on stable.
+ // Allow them to use `feature(test)`, but nothing else.
+ rustflags.arg("-Zallow-features=binary-dep-depinfo,test,backtrace,proc_macro_internals,proc_macro_diagnostic,proc_macro_span");
+ }
+ Mode::Std | Mode::Rustc | Mode::Codegen | Mode::ToolRustc => {}
+ }
+
+ cargo.arg("-j").arg(self.jobs().to_string());
+ // Remove make-related flags to ensure Cargo can correctly set things up
+ cargo.env_remove("MAKEFLAGS");
+ cargo.env_remove("MFLAGS");
+
+ // FIXME: Temporary fix for https://github.com/rust-lang/cargo/issues/3005
+ // Force cargo to output binaries with disambiguating hashes in the name
+ let mut metadata = if compiler.stage == 0 {
+ // Treat stage0 like a special channel, whether it's a normal prior-
+ // release rustc or a local rebuild with the same version, so we
+ // never mix these libraries by accident.
+ "bootstrap".to_string()
+ } else {
+ self.config.channel.to_string()
+ };
+ // We want to make sure that none of the dependencies between
+ // std/test/rustc unify with one another. This is done for weird linkage
+ // reasons but the gist of the problem is that if librustc, libtest, and
+ // libstd all depend on libc from crates.io (which they actually do) we
+ // want to make sure they all get distinct versions. Things get really
+ // weird if we try to unify all these dependencies right now, namely
+ // around how many times the library is linked in dynamic libraries and
+ // such. If rustc were a static executable or if we didn't ship dylibs
+ // this wouldn't be a problem, but we do, so it is. This is in general
+ // just here to make sure things build right. If you can remove this and
+ // things still build right, please do!
+ match mode {
+ Mode::Std => metadata.push_str("std"),
+ // When we're building rustc tools, they're built with a search path
+ // that contains things built during the rustc build. For example,
+ // bitflags is built during the rustc build, and is a dependency of
+ // rustdoc as well. We're building rustdoc in a different target
+ // directory, though, which means that Cargo will rebuild the
+ // dependency. When we go on to build rustdoc, we'll look for
+ // bitflags, and find two different copies: one built during the
+ // rustc step and one that we just built. This isn't always a
+ // problem, somehow -- not really clear why -- but we know that this
+ // fixes things.
+ Mode::ToolRustc => metadata.push_str("tool-rustc"),
+ // Same for codegen backends.
+ Mode::Codegen => metadata.push_str("codegen"),
+ _ => {}
+ }
+ cargo.env("__CARGO_DEFAULT_LIB_METADATA", &metadata);
+
+ if cmd == "clippy" {
+ rustflags.arg("-Zforce-unstable-if-unmarked");
+ }
+
+ rustflags.arg("-Zmacro-backtrace");
+
+ let want_rustdoc = self.doc_tests != DocTests::No;
+
+ // We synthetically interpret a stage0 compiler used to build tools as a
+ // "raw" compiler in that it's the exact snapshot we download. Normally
+ // the stage0 build means it uses libraries build by the stage0
+ // compiler, but for tools we just use the precompiled libraries that
+ // we've downloaded
+ let use_snapshot = mode == Mode::ToolBootstrap;
+ assert!(!use_snapshot || stage == 0 || self.local_rebuild);
+
+ let maybe_sysroot = self.sysroot(compiler);
+ let sysroot = if use_snapshot { self.rustc_snapshot_sysroot() } else { &maybe_sysroot };
+ let libdir = self.rustc_libdir(compiler);
+
+ // Clear the output directory if the real rustc we're using has changed;
+ // Cargo cannot detect this as it thinks rustc is bootstrap/debug/rustc.
+ //
+ // Avoid doing this during dry run as that usually means the relevant
+ // compiler is not yet linked/copied properly.
+ //
+ // Only clear out the directory if we're compiling std; otherwise, we
+ // should let Cargo take care of things for us (via depdep info)
+ if !self.config.dry_run && mode == Mode::Std && cmd == "build" {
+ self.clear_if_dirty(&out_dir, &self.rustc(compiler));
+ }
+
+ // Customize the compiler we're running. Specify the compiler to cargo
+ // as our shim and then pass it some various options used to configure
+ // how the actual compiler itself is called.
+ //
+ // These variables are primarily all read by
+ // src/bootstrap/bin/{rustc.rs,rustdoc.rs}
+ cargo
+ .env("RUSTBUILD_NATIVE_DIR", self.native_dir(target))
+ .env("RUSTC_REAL", self.rustc(compiler))
+ .env("RUSTC_STAGE", stage.to_string())
+ .env("RUSTC_SYSROOT", &sysroot)
+ .env("RUSTC_LIBDIR", &libdir)
+ .env("RUSTDOC", self.bootstrap_out.join("rustdoc"))
+ .env(
+ "RUSTDOC_REAL",
+ if cmd == "doc" || cmd == "rustdoc" || (cmd == "test" && want_rustdoc) {
+ self.rustdoc(compiler)
+ } else {
+ PathBuf::from("/path/to/nowhere/rustdoc/not/required")
+ },
+ )
+ .env("RUSTC_ERROR_METADATA_DST", self.extended_error_dir())
+ .env("RUSTC_BREAK_ON_ICE", "1");
+ // Clippy support is a hack and uses the default `cargo-clippy` in path.
+ // Don't override RUSTC so that the `cargo-clippy` in path will be run.
+ if cmd != "clippy" {
+ cargo.env("RUSTC", self.bootstrap_out.join("rustc"));
+ }
+
+ // Dealing with rpath here is a little special, so let's go into some
+ // detail. First off, `-rpath` is a linker option on Unix platforms
+ // which adds to the runtime dynamic loader path when looking for
+ // dynamic libraries. We use this by default on Unix platforms to ensure
+ // that our nightlies behave the same on Windows, that is they work out
+ // of the box. This can be disabled, of course, but basically that's why
+ // we're gated on RUSTC_RPATH here.
+ //
+ // Ok, so the astute might be wondering "why isn't `-C rpath` used
+ // here?" and that is indeed a good question to ask. This codegen
+ // option is the compiler's current interface to generating an rpath.
+ // Unfortunately it doesn't quite suffice for us. The flag currently
+ // takes no value as an argument, so the compiler calculates what it
+ // should pass to the linker as `-rpath`. This unfortunately is based on
+ // the **compile time** directory structure which when building with
+ // Cargo will be very different than the runtime directory structure.
+ //
+ // All that's a really long winded way of saying that if we use
+ // `-Crpath` then the executables generated have the wrong rpath of
+ // something like `$ORIGIN/deps` when in fact the way we distribute
+ // rustc requires the rpath to be `$ORIGIN/../lib`.
+ //
+ // So, all in all, to set up the correct rpath we pass the linker
+ // argument manually via `-C link-args=-Wl,-rpath,...`. Plus isn't it
+ // fun to pass a flag to a tool to pass a flag to pass a flag to a tool
+ // to change a flag in a binary?
+ if self.config.rust_rpath && util::use_host_linker(target) {
+ let rpath = if target.contains("apple") {
+ // Note that we need to take one extra step on macOS to also pass
+ // `-Wl,-instal_name,@rpath/...` to get things to work right. To
+ // do that we pass a weird flag to the compiler to get it to do
+ // so. Note that this is definitely a hack, and we should likely
+ // flesh out rpath support more fully in the future.
+ rustflags.arg("-Zosx-rpath-install-name");
+ Some("-Wl,-rpath,@loader_path/../lib")
+ } else if !target.contains("windows") {
+ rustflags.arg("-Clink-args=-Wl,-z,origin");
+ Some("-Wl,-rpath,$ORIGIN/../lib")
+ } else {
+ None
+ };
+ if let Some(rpath) = rpath {
+ rustflags.arg(&format!("-Clink-args={}", rpath));
+ }
+ }
+
+ if let Some(host_linker) = self.linker(compiler.host) {
+ cargo.env("RUSTC_HOST_LINKER", host_linker);
+ }
+ if self.is_fuse_ld_lld(compiler.host) {
+ cargo.env("RUSTC_HOST_FUSE_LD_LLD", "1");
+ cargo.env("RUSTDOC_FUSE_LD_LLD", "1");
+ }
+
+ if let Some(target_linker) = self.linker(target) {
+ let target = crate::envify(&target.triple);
+ cargo.env(&format!("CARGO_TARGET_{}_LINKER", target), target_linker);
+ }
+ if self.is_fuse_ld_lld(target) {
+ rustflags.arg("-Clink-args=-fuse-ld=lld");
+ }
+ self.lld_flags(target).for_each(|flag| {
+ rustdocflags.arg(&flag);
+ });
+
+ if !(["build", "check", "clippy", "fix", "rustc"].contains(&cmd)) && want_rustdoc {
+ cargo.env("RUSTDOC_LIBDIR", self.rustc_libdir(compiler));
+ }
+
+ let debuginfo_level = match mode {
+ Mode::Rustc | Mode::Codegen => self.config.rust_debuginfo_level_rustc,
+ Mode::Std => self.config.rust_debuginfo_level_std,
+ Mode::ToolBootstrap | Mode::ToolStd | Mode::ToolRustc => {
+ self.config.rust_debuginfo_level_tools
+ }
+ };
+ cargo.env(profile_var("DEBUG"), debuginfo_level.to_string());
+ cargo.env(
+ profile_var("DEBUG_ASSERTIONS"),
+ if mode == Mode::Std {
+ self.config.rust_debug_assertions_std.to_string()
+ } else {
+ self.config.rust_debug_assertions.to_string()
+ },
+ );
+ cargo.env(
+ profile_var("OVERFLOW_CHECKS"),
+ if mode == Mode::Std {
+ self.config.rust_overflow_checks_std.to_string()
+ } else {
+ self.config.rust_overflow_checks.to_string()
+ },
+ );
+
+ if !target.contains("windows") {
+ let needs_unstable_opts = target.contains("linux")
+ || target.contains("solaris")
+ || target.contains("windows")
+ || target.contains("bsd")
+ || target.contains("dragonfly")
+ || target.contains("illumos");
+
+ if needs_unstable_opts {
+ rustflags.arg("-Zunstable-options");
+ }
+ match self.config.rust_split_debuginfo {
+ SplitDebuginfo::Packed => rustflags.arg("-Csplit-debuginfo=packed"),
+ SplitDebuginfo::Unpacked => rustflags.arg("-Csplit-debuginfo=unpacked"),
+ SplitDebuginfo::Off => rustflags.arg("-Csplit-debuginfo=off"),
+ };
+ }
+
+ if self.config.cmd.bless() {
+ // Bless `expect!` tests.
+ cargo.env("UPDATE_EXPECT", "1");
+ }
+
+ if !mode.is_tool() {
+ cargo.env("RUSTC_FORCE_UNSTABLE", "1");
+ }
+
+ if let Some(x) = self.crt_static(target) {
+ if x {
+ rustflags.arg("-Ctarget-feature=+crt-static");
+ } else {
+ rustflags.arg("-Ctarget-feature=-crt-static");
+ }
+ }
+
+ if let Some(x) = self.crt_static(compiler.host) {
+ cargo.env("RUSTC_HOST_CRT_STATIC", x.to_string());
+ }
+
+ if let Some(map_to) = self.build.debuginfo_map_to(GitRepo::Rustc) {
+ let map = format!("{}={}", self.build.src.display(), map_to);
+ cargo.env("RUSTC_DEBUGINFO_MAP", map);
+
+ // `rustc` needs to know the virtual `/rustc/$hash` we're mapping to,
+ // in order to opportunistically reverse it later.
+ cargo.env("CFG_VIRTUAL_RUST_SOURCE_BASE_DIR", map_to);
+ }
+
+ // Enable usage of unstable features
+ cargo.env("RUSTC_BOOTSTRAP", "1");
+ self.add_rust_test_threads(&mut cargo);
+
+ // Almost all of the crates that we compile as part of the bootstrap may
+ // have a build script, including the standard library. To compile a
+ // build script, however, it itself needs a standard library! This
+ // introduces a bit of a pickle when we're compiling the standard
+ // library itself.
+ //
+ // To work around this we actually end up using the snapshot compiler
+ // (stage0) for compiling build scripts of the standard library itself.
+ // The stage0 compiler is guaranteed to have a libstd available for use.
+ //
+ // For other crates, however, we know that we've already got a standard
+ // library up and running, so we can use the normal compiler to compile
+ // build scripts in that situation.
+ if mode == Mode::Std {
+ cargo
+ .env("RUSTC_SNAPSHOT", &self.initial_rustc)
+ .env("RUSTC_SNAPSHOT_LIBDIR", self.rustc_snapshot_libdir());
+ } else {
+ cargo
+ .env("RUSTC_SNAPSHOT", self.rustc(compiler))
+ .env("RUSTC_SNAPSHOT_LIBDIR", self.rustc_libdir(compiler));
+ }
+
+ // Tools that use compiler libraries may inherit the `-lLLVM` link
+ // requirement, but the `-L` library path is not propagated across
+ // separate Cargo projects. We can add LLVM's library path to the
+ // platform-specific environment variable as a workaround.
+ if mode == Mode::ToolRustc || mode == Mode::Codegen {
+ if let Some(llvm_config) = self.llvm_config(target) {
+ let llvm_libdir = output(Command::new(&llvm_config).arg("--libdir"));
+ add_link_lib_path(vec![llvm_libdir.trim().into()], &mut cargo);
+ }
+ }
+
+ // Compile everything except libraries and proc macros with the more
+ // efficient initial-exec TLS model. This doesn't work with `dlopen`,
+ // so we can't use it by default in general, but we can use it for tools
+ // and our own internal libraries.
+ if !mode.must_support_dlopen() && !target.triple.starts_with("powerpc-") {
+ rustflags.arg("-Ztls-model=initial-exec");
+ }
+
+ if self.config.incremental {
+ cargo.env("CARGO_INCREMENTAL", "1");
+ } else {
+ // Don't rely on any default setting for incr. comp. in Cargo
+ cargo.env("CARGO_INCREMENTAL", "0");
+ }
+
+ if let Some(ref on_fail) = self.config.on_fail {
+ cargo.env("RUSTC_ON_FAIL", on_fail);
+ }
+
+ if self.config.print_step_timings {
+ cargo.env("RUSTC_PRINT_STEP_TIMINGS", "1");
+ }
+
+ if self.config.print_step_rusage {
+ cargo.env("RUSTC_PRINT_STEP_RUSAGE", "1");
+ }
+
+ if self.config.backtrace_on_ice {
+ cargo.env("RUSTC_BACKTRACE_ON_ICE", "1");
+ }
+
+ cargo.env("RUSTC_VERBOSE", self.verbosity.to_string());
+
+ if source_type == SourceType::InTree {
+ let mut lint_flags = Vec::new();
+ // When extending this list, add the new lints to the RUSTFLAGS of the
+ // build_bootstrap function of src/bootstrap/bootstrap.py as well as
+ // some code doesn't go through this `rustc` wrapper.
+ lint_flags.push("-Wrust_2018_idioms");
+ lint_flags.push("-Wunused_lifetimes");
+ lint_flags.push("-Wsemicolon_in_expressions_from_macros");
+
+ if self.config.deny_warnings {
+ lint_flags.push("-Dwarnings");
+ rustdocflags.arg("-Dwarnings");
+ }
+
+ // This does not use RUSTFLAGS due to caching issues with Cargo.
+ // Clippy is treated as an "in tree" tool, but shares the same
+ // cache as other "submodule" tools. With these options set in
+ // RUSTFLAGS, that causes *every* shared dependency to be rebuilt.
+ // By injecting this into the rustc wrapper, this circumvents
+ // Cargo's fingerprint detection. This is fine because lint flags
+ // are always ignored in dependencies. Eventually this should be
+ // fixed via better support from Cargo.
+ cargo.env("RUSTC_LINT_FLAGS", lint_flags.join(" "));
+
+ rustdocflags.arg("-Wrustdoc::invalid_codeblock_attributes");
+ }
+
+ if mode == Mode::Rustc {
+ rustflags.arg("-Zunstable-options");
+ rustflags.arg("-Wrustc::internal");
+ }
+
+ // Throughout the build Cargo can execute a number of build scripts
+ // compiling C/C++ code and we need to pass compilers, archivers, flags, etc
+ // obtained previously to those build scripts.
+ // Build scripts use either the `cc` crate or `configure/make` so we pass
+ // the options through environment variables that are fetched and understood by both.
+ //
+ // FIXME: the guard against msvc shouldn't need to be here
+ if target.contains("msvc") {
+ if let Some(ref cl) = self.config.llvm_clang_cl {
+ cargo.env("CC", cl).env("CXX", cl);
+ }
+ } else {
+ let ccache = self.config.ccache.as_ref();
+ let ccacheify = |s: &Path| {
+ let ccache = match ccache {
+ Some(ref s) => s,
+ None => return s.display().to_string(),
+ };
+ // FIXME: the cc-rs crate only recognizes the literal strings
+ // `ccache` and `sccache` when doing caching compilations, so we
+ // mirror that here. It should probably be fixed upstream to
+ // accept a new env var or otherwise work with custom ccache
+ // vars.
+ match &ccache[..] {
+ "ccache" | "sccache" => format!("{} {}", ccache, s.display()),
+ _ => s.display().to_string(),
+ }
+ };
+ let cc = ccacheify(&self.cc(target));
+ cargo.env(format!("CC_{}", target.triple), &cc);
+
+ let cflags = self.cflags(target, GitRepo::Rustc, CLang::C).join(" ");
+ cargo.env(format!("CFLAGS_{}", target.triple), &cflags);
+
+ if let Some(ar) = self.ar(target) {
+ let ranlib = format!("{} s", ar.display());
+ cargo
+ .env(format!("AR_{}", target.triple), ar)
+ .env(format!("RANLIB_{}", target.triple), ranlib);
+ }
+
+ if let Ok(cxx) = self.cxx(target) {
+ let cxx = ccacheify(&cxx);
+ let cxxflags = self.cflags(target, GitRepo::Rustc, CLang::Cxx).join(" ");
+ cargo
+ .env(format!("CXX_{}", target.triple), &cxx)
+ .env(format!("CXXFLAGS_{}", target.triple), cxxflags);
+ }
+ }
+
+ if mode == Mode::Std && self.config.extended && compiler.is_final_stage(self) {
+ rustflags.arg("-Zsave-analysis");
+ cargo.env(
+ "RUST_SAVE_ANALYSIS_CONFIG",
+ "{\"output_file\": null,\"full_docs\": false,\
+ \"pub_only\": true,\"reachable_only\": false,\
+ \"distro_crate\": true,\"signatures\": false,\"borrow_data\": false}",
+ );
+ }
+
+ // If Control Flow Guard is enabled, pass the `control-flow-guard` flag to rustc
+ // when compiling the standard library, since this might be linked into the final outputs
+ // produced by rustc. Since this mitigation is only available on Windows, only enable it
+ // for the standard library in case the compiler is run on a non-Windows platform.
+ // This is not needed for stage 0 artifacts because these will only be used for building
+ // the stage 1 compiler.
+ if cfg!(windows)
+ && mode == Mode::Std
+ && self.config.control_flow_guard
+ && compiler.stage >= 1
+ {
+ rustflags.arg("-Ccontrol-flow-guard");
+ }
+
+ // For `cargo doc` invocations, make rustdoc print the Rust version into the docs
+ // This replaces spaces with newlines because RUSTDOCFLAGS does not
+ // support arguments with regular spaces. Hopefully someday Cargo will
+ // have space support.
+ let rust_version = self.rust_version().replace(' ', "\n");
+ rustdocflags.arg("--crate-version").arg(&rust_version);
+
+ // Environment variables *required* throughout the build
+ //
+ // FIXME: should update code to not require this env var
+ cargo.env("CFG_COMPILER_HOST_TRIPLE", target.triple);
+
+ // Set this for all builds to make sure doc builds also get it.
+ cargo.env("CFG_RELEASE_CHANNEL", &self.config.channel);
+
+ // This one's a bit tricky. As of the time of this writing the compiler
+ // links to the `winapi` crate on crates.io. This crate provides raw
+ // bindings to Windows system functions, sort of like libc does for
+ // Unix. This crate also, however, provides "import libraries" for the
+ // MinGW targets. There's an import library per dll in the windows
+ // distribution which is what's linked to. These custom import libraries
+ // are used because the winapi crate can reference Windows functions not
+ // present in the MinGW import libraries.
+ //
+ // For example MinGW may ship libdbghelp.a, but it may not have
+ // references to all the functions in the dbghelp dll. Instead the
+ // custom import library for dbghelp in the winapi crates has all this
+ // information.
+ //
+ // Unfortunately for us though the import libraries are linked by
+ // default via `-ldylib=winapi_foo`. That is, they're linked with the
+ // `dylib` type with a `winapi_` prefix (so the winapi ones don't
+ // conflict with the system MinGW ones). This consequently means that
+ // the binaries we ship of things like rustc_codegen_llvm (aka the rustc_codegen_llvm
+ // DLL) when linked against *again*, for example with procedural macros
+ // or plugins, will trigger the propagation logic of `-ldylib`, passing
+ // `-lwinapi_foo` to the linker again. This isn't actually available in
+ // our distribution, however, so the link fails.
+ //
+ // To solve this problem we tell winapi to not use its bundled import
+ // libraries. This means that it will link to the system MinGW import
+ // libraries by default, and the `-ldylib=foo` directives will still get
+ // passed to the final linker, but they'll look like `-lfoo` which can
+ // be resolved because MinGW has the import library. The downside is we
+ // don't get newer functions from Windows, but we don't use any of them
+ // anyway.
+ if !mode.is_tool() {
+ cargo.env("WINAPI_NO_BUNDLED_LIBRARIES", "1");
+ }
+
+ for _ in 0..self.verbosity {
+ cargo.arg("-v");
+ }
+
+ match (mode, self.config.rust_codegen_units_std, self.config.rust_codegen_units) {
+ (Mode::Std, Some(n), _) | (_, _, Some(n)) => {
+ cargo.env(profile_var("CODEGEN_UNITS"), n.to_string());
+ }
+ _ => {
+ // Don't set anything
+ }
+ }
+
+ if self.config.rust_optimize {
+ // FIXME: cargo bench/install do not accept `--release`
+ if cmd != "bench" && cmd != "install" {
+ cargo.arg("--release");
+ }
+ }
+
+ if self.config.locked_deps {
+ cargo.arg("--locked");
+ }
+ if self.config.vendor || self.is_sudo {
+ cargo.arg("--frozen");
+ }
+
+ // Try to use a sysroot-relative bindir, in case it was configured absolutely.
+ cargo.env("RUSTC_INSTALL_BINDIR", self.config.bindir_relative());
+
+ self.ci_env.force_coloring_in_ci(&mut cargo);
+
+ // When we build Rust dylibs they're all intended for intermediate
+ // usage, so make sure we pass the -Cprefer-dynamic flag instead of
+ // linking all deps statically into the dylib.
+ if matches!(mode, Mode::Std | Mode::Rustc) {
+ rustflags.arg("-Cprefer-dynamic");
+ }
+
+ // When building incrementally we default to a lower ThinLTO import limit
+ // (unless explicitly specified otherwise). This will produce a somewhat
+ // slower code but give way better compile times.
+ {
+ let limit = match self.config.rust_thin_lto_import_instr_limit {
+ Some(limit) => Some(limit),
+ None if self.config.incremental => Some(10),
+ _ => None,
+ };
+
+ if let Some(limit) = limit {
+ rustflags.arg(&format!("-Cllvm-args=-import-instr-limit={}", limit));
+ }
+ }
+
+ Cargo { command: cargo, rustflags, rustdocflags }
+ }
+
+ /// Ensure that a given step is built, returning its output. This will
+ /// cache the step, so it is safe (and good!) to call this as often as
+ /// needed to ensure that all dependencies are built.
+ pub fn ensure<S: Step>(&'a self, step: S) -> S::Output {
+ {
+ let mut stack = self.stack.borrow_mut();
+ for stack_step in stack.iter() {
+ // should skip
+ if stack_step.downcast_ref::<S>().map_or(true, |stack_step| *stack_step != step) {
+ continue;
+ }
+ let mut out = String::new();
+ out += &format!("\n\nCycle in build detected when adding {:?}\n", step);
+ for el in stack.iter().rev() {
+ out += &format!("\t{:?}\n", el);
+ }
+ panic!("{}", out);
+ }
+ if let Some(out) = self.cache.get(&step) {
+ self.verbose_than(1, &format!("{}c {:?}", " ".repeat(stack.len()), step));
+
+ return out;
+ }
+ self.verbose_than(1, &format!("{}> {:?}", " ".repeat(stack.len()), step));
+ stack.push(Box::new(step.clone()));
+ }
+
+ #[cfg(feature = "build-metrics")]
+ self.metrics.enter_step(&step);
+
+ let (out, dur) = {
+ let start = Instant::now();
+ let zero = Duration::new(0, 0);
+ let parent = self.time_spent_on_dependencies.replace(zero);
+ let out = step.clone().run(self);
+ let dur = start.elapsed();
+ let deps = self.time_spent_on_dependencies.replace(parent + dur);
+ (out, dur - deps)
+ };
+
+ if self.config.print_step_timings && !self.config.dry_run {
+ let step_string = format!("{:?}", step);
+ let brace_index = step_string.find("{").unwrap_or(0);
+ let type_string = type_name::<S>();
+ println!(
+ "[TIMING] {} {} -- {}.{:03}",
+ &type_string.strip_prefix("bootstrap::").unwrap_or(type_string),
+ &step_string[brace_index..],
+ dur.as_secs(),
+ dur.subsec_millis()
+ );
+ }
+
+ #[cfg(feature = "build-metrics")]
+ self.metrics.exit_step();
+
+ {
+ let mut stack = self.stack.borrow_mut();
+ let cur_step = stack.pop().expect("step stack empty");
+ assert_eq!(cur_step.downcast_ref(), Some(&step));
+ }
+ self.verbose_than(1, &format!("{}< {:?}", " ".repeat(self.stack.borrow().len()), step));
+ self.cache.put(step, out.clone());
+ out
+ }
+
+ /// Ensure that a given step is built *only if it's supposed to be built by default*, returning
+ /// its output. This will cache the step, so it's safe (and good!) to call this as often as
+ /// needed to ensure that all dependencies are build.
+ pub(crate) fn ensure_if_default<T, S: Step<Output = Option<T>>>(
+ &'a self,
+ step: S,
+ kind: Kind,
+ ) -> S::Output {
+ let desc = StepDescription::from::<S>(kind);
+ let should_run = (desc.should_run)(ShouldRun::new(self, desc.kind));
+
+ // Avoid running steps contained in --exclude
+ for pathset in &should_run.paths {
+ if desc.is_excluded(self, pathset) {
+ return None;
+ }
+ }
+
+ // Only execute if it's supposed to run as default
+ if desc.default && should_run.is_really_default() { self.ensure(step) } else { None }
+ }
+
+ /// Checks if any of the "should_run" paths is in the `Builder` paths.
+ pub(crate) fn was_invoked_explicitly<S: Step>(&'a self, kind: Kind) -> bool {
+ let desc = StepDescription::from::<S>(kind);
+ let should_run = (desc.should_run)(ShouldRun::new(self, desc.kind));
+
+ for path in &self.paths {
+ if should_run.paths.iter().any(|s| s.has(path, Some(desc.kind)))
+ && !desc.is_excluded(
+ self,
+ &PathSet::Suite(TaskPath { path: path.clone(), kind: Some(desc.kind) }),
+ )
+ {
+ return true;
+ }
+ }
+
+ false
+ }
+}
+
+#[cfg(test)]
+mod tests;
+
+#[derive(Debug, Clone)]
+struct Rustflags(String, TargetSelection);
+
+impl Rustflags {
+ fn new(target: TargetSelection) -> Rustflags {
+ let mut ret = Rustflags(String::new(), target);
+ ret.propagate_cargo_env("RUSTFLAGS");
+ ret
+ }
+
+ /// By default, cargo will pick up on various variables in the environment. However, bootstrap
+ /// reuses those variables to pass additional flags to rustdoc, so by default they get overridden.
+ /// Explicitly add back any previous value in the environment.
+ ///
+ /// `prefix` is usually `RUSTFLAGS` or `RUSTDOCFLAGS`.
+ fn propagate_cargo_env(&mut self, prefix: &str) {
+ // Inherit `RUSTFLAGS` by default ...
+ self.env(prefix);
+
+ // ... and also handle target-specific env RUSTFLAGS if they're configured.
+ let target_specific = format!("CARGO_TARGET_{}_{}", crate::envify(&self.1.triple), prefix);
+ self.env(&target_specific);
+ }
+
+ fn env(&mut self, env: &str) {
+ if let Ok(s) = env::var(env) {
+ for part in s.split(' ') {
+ self.arg(part);
+ }
+ }
+ }
+
+ fn arg(&mut self, arg: &str) -> &mut Self {
+ assert_eq!(arg.split(' ').count(), 1);
+ if !self.0.is_empty() {
+ self.0.push(' ');
+ }
+ self.0.push_str(arg);
+ self
+ }
+}
+
+#[derive(Debug)]
+pub struct Cargo {
+ command: Command,
+ rustflags: Rustflags,
+ rustdocflags: Rustflags,
+}
+
+impl Cargo {
+ pub fn rustdocflag(&mut self, arg: &str) -> &mut Cargo {
+ self.rustdocflags.arg(arg);
+ self
+ }
+ pub fn rustflag(&mut self, arg: &str) -> &mut Cargo {
+ self.rustflags.arg(arg);
+ self
+ }
+
+ pub fn arg(&mut self, arg: impl AsRef<OsStr>) -> &mut Cargo {
+ self.command.arg(arg.as_ref());
+ self
+ }
+
+ pub fn args<I, S>(&mut self, args: I) -> &mut Cargo
+ where
+ I: IntoIterator<Item = S>,
+ S: AsRef<OsStr>,
+ {
+ for arg in args {
+ self.arg(arg.as_ref());
+ }
+ self
+ }
+
+ pub fn env(&mut self, key: impl AsRef<OsStr>, value: impl AsRef<OsStr>) -> &mut Cargo {
+ // These are managed through rustflag/rustdocflag interfaces.
+ assert_ne!(key.as_ref(), "RUSTFLAGS");
+ assert_ne!(key.as_ref(), "RUSTDOCFLAGS");
+ self.command.env(key.as_ref(), value.as_ref());
+ self
+ }
+
+ pub fn add_rustc_lib_path(&mut self, builder: &Builder<'_>, compiler: Compiler) {
+ builder.add_rustc_lib_path(compiler, &mut self.command);
+ }
+
+ pub fn current_dir(&mut self, dir: &Path) -> &mut Cargo {
+ self.command.current_dir(dir);
+ self
+ }
+}
+
+impl From<Cargo> for Command {
+ fn from(mut cargo: Cargo) -> Command {
+ let rustflags = &cargo.rustflags.0;
+ if !rustflags.is_empty() {
+ cargo.command.env("RUSTFLAGS", rustflags);
+ }
+
+ let rustdocflags = &cargo.rustdocflags.0;
+ if !rustdocflags.is_empty() {
+ cargo.command.env("RUSTDOCFLAGS", rustdocflags);
+ }
+
+ cargo.command
+ }
+}
diff --git a/src/bootstrap/builder/tests.rs b/src/bootstrap/builder/tests.rs
new file mode 100644
index 000000000..c084e77d3
--- /dev/null
+++ b/src/bootstrap/builder/tests.rs
@@ -0,0 +1,671 @@
+use super::*;
+use crate::config::{Config, TargetSelection};
+use std::thread;
+
+fn configure(cmd: &str, host: &[&str], target: &[&str]) -> Config {
+ configure_with_args(&[cmd.to_owned()], host, target)
+}
+
+fn configure_with_args(cmd: &[String], host: &[&str], target: &[&str]) -> Config {
+ let mut config = Config::parse(cmd);
+ // don't save toolstates
+ config.save_toolstates = None;
+ config.dry_run = true;
+
+ // Ignore most submodules, since we don't need them for a dry run.
+ // But make sure to check out the `doc` and `rust-analyzer` submodules, since some steps need them
+ // just to know which commands to run.
+ let submodule_build = Build::new(Config {
+ // don't include LLVM, so CI doesn't require ninja/cmake to be installed
+ rust_codegen_backends: vec![],
+ ..Config::parse(&["check".to_owned()])
+ });
+ submodule_build.update_submodule(Path::new("src/doc/book"));
+ submodule_build.update_submodule(Path::new("src/tools/rust-analyzer"));
+ config.submodules = Some(false);
+
+ config.ninja_in_file = false;
+ // try to avoid spurious failures in dist where we create/delete each others file
+ // HACK: rather than pull in `tempdir`, use the one that cargo has conveniently created for us
+ let dir = Path::new(env!("OUT_DIR"))
+ .join("tmp-rustbuild-tests")
+ .join(&thread::current().name().unwrap_or("unknown").replace(":", "-"));
+ t!(fs::create_dir_all(&dir));
+ config.out = dir;
+ config.build = TargetSelection::from_user("A");
+ config.hosts = host.iter().map(|s| TargetSelection::from_user(s)).collect();
+ config.targets = target.iter().map(|s| TargetSelection::from_user(s)).collect();
+ config
+}
+
+fn first<A, B>(v: Vec<(A, B)>) -> Vec<A> {
+ v.into_iter().map(|(a, _)| a).collect::<Vec<_>>()
+}
+
+fn run_build(paths: &[PathBuf], config: Config) -> Cache {
+ let kind = config.cmd.kind();
+ let build = Build::new(config);
+ let builder = Builder::new(&build);
+ builder.run_step_descriptions(&Builder::get_step_descriptions(kind), paths);
+ builder.cache
+}
+
+fn check_cli<const N: usize>(paths: [&str; N]) {
+ run_build(
+ &paths.map(PathBuf::from),
+ configure_with_args(&paths.map(String::from), &["A"], &["A"]),
+ );
+}
+
+macro_rules! std {
+ ($host:ident => $target:ident, stage = $stage:literal) => {
+ compile::Std::new(
+ Compiler { host: TargetSelection::from_user(stringify!($host)), stage: $stage },
+ TargetSelection::from_user(stringify!($target)),
+ )
+ };
+}
+
+macro_rules! rustc {
+ ($host:ident => $target:ident, stage = $stage:literal) => {
+ compile::Rustc::new(
+ Compiler { host: TargetSelection::from_user(stringify!($host)), stage: $stage },
+ TargetSelection::from_user(stringify!($target)),
+ )
+ };
+}
+
+#[test]
+fn test_valid() {
+ // make sure multi suite paths are accepted
+ check_cli(["test", "src/test/ui/attr-start.rs", "src/test/ui/attr-shebang.rs"]);
+}
+
+#[test]
+#[should_panic]
+fn test_invalid() {
+ // make sure that invalid paths are caught, even when combined with valid paths
+ check_cli(["test", "library/std", "x"]);
+}
+
+#[test]
+fn test_intersection() {
+ let set = PathSet::Set(
+ ["library/core", "library/alloc", "library/std"].into_iter().map(TaskPath::parse).collect(),
+ );
+ let mut command_paths =
+ vec![Path::new("library/core"), Path::new("library/alloc"), Path::new("library/stdarch")];
+ let subset = set.intersection_removing_matches(&mut command_paths, None);
+ assert_eq!(
+ subset,
+ PathSet::Set(["library/core", "library/alloc"].into_iter().map(TaskPath::parse).collect())
+ );
+ assert_eq!(command_paths, vec![Path::new("library/stdarch")]);
+}
+
+#[test]
+fn test_exclude() {
+ let mut config = configure("test", &["A"], &["A"]);
+ config.exclude = vec![TaskPath::parse("src/tools/tidy")];
+ let cache = run_build(&[], config);
+
+ // Ensure we have really excluded tidy
+ assert!(!cache.contains::<test::Tidy>());
+
+ // Ensure other tests are not affected.
+ assert!(cache.contains::<test::RustdocUi>());
+}
+
+#[test]
+fn test_exclude_kind() {
+ let path = PathBuf::from("src/tools/cargotest");
+ let exclude = TaskPath::parse("test::src/tools/cargotest");
+ assert_eq!(exclude, TaskPath { kind: Some(Kind::Test), path: path.clone() });
+
+ let mut config = configure("test", &["A"], &["A"]);
+ // Ensure our test is valid, and `test::Cargotest` would be run without the exclude.
+ assert!(run_build(&[path.clone()], config.clone()).contains::<test::Cargotest>());
+ // Ensure tests for cargotest are skipped.
+ config.exclude = vec![exclude.clone()];
+ assert!(!run_build(&[path.clone()], config).contains::<test::Cargotest>());
+
+ // Ensure builds for cargotest are not skipped.
+ let mut config = configure("build", &["A"], &["A"]);
+ config.exclude = vec![exclude];
+ assert!(run_build(&[path], config).contains::<tool::CargoTest>());
+}
+
+/// Ensure that if someone passes both a single crate and `library`, all library crates get built.
+#[test]
+fn alias_and_path_for_library() {
+ let mut cache =
+ run_build(&["library".into(), "core".into()], configure("build", &["A"], &["A"]));
+ assert_eq!(
+ first(cache.all::<compile::Std>()),
+ &[std!(A => A, stage = 0), std!(A => A, stage = 1)]
+ );
+}
+
+mod defaults {
+ use super::{configure, first, run_build};
+ use crate::builder::*;
+ use crate::Config;
+ use pretty_assertions::assert_eq;
+
+ #[test]
+ fn build_default() {
+ let mut cache = run_build(&[], configure("build", &["A"], &["A"]));
+
+ let a = TargetSelection::from_user("A");
+ assert_eq!(
+ first(cache.all::<compile::Std>()),
+ &[std!(A => A, stage = 0), std!(A => A, stage = 1),]
+ );
+ assert!(!cache.all::<compile::Assemble>().is_empty());
+ // Make sure rustdoc is only built once.
+ assert_eq!(
+ first(cache.all::<tool::Rustdoc>()),
+ // Recall that rustdoc stages are off-by-one
+ // - this is the compiler it's _linked_ to, not built with.
+ &[tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }],
+ );
+ assert_eq!(first(cache.all::<compile::Rustc>()), &[rustc!(A => A, stage = 0)],);
+ }
+
+ #[test]
+ fn build_stage_0() {
+ let config = Config { stage: 0, ..configure("build", &["A"], &["A"]) };
+ let mut cache = run_build(&[], config);
+
+ let a = TargetSelection::from_user("A");
+ assert_eq!(first(cache.all::<compile::Std>()), &[std!(A => A, stage = 0)]);
+ assert!(!cache.all::<compile::Assemble>().is_empty());
+ assert_eq!(
+ first(cache.all::<tool::Rustdoc>()),
+ // This is the beta rustdoc.
+ // Add an assert here to make sure this is the only rustdoc built.
+ &[tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } }],
+ );
+ assert!(cache.all::<compile::Rustc>().is_empty());
+ }
+
+ #[test]
+ fn build_cross_compile() {
+ let config = Config { stage: 1, ..configure("build", &["A", "B"], &["A", "B"]) };
+ let mut cache = run_build(&[], config);
+
+ let a = TargetSelection::from_user("A");
+ let b = TargetSelection::from_user("B");
+
+ // Ideally, this build wouldn't actually have `target: a`
+ // rustdoc/rustcc/std here (the user only requested a host=B build, so
+ // there's not really a need for us to build for target A in this case
+ // (since we're producing stage 1 libraries/binaries). But currently
+ // rustbuild is just a bit buggy here; this should be fixed though.
+ assert_eq!(
+ first(cache.all::<compile::Std>()),
+ &[
+ std!(A => A, stage = 0),
+ std!(A => A, stage = 1),
+ std!(A => B, stage = 0),
+ std!(A => B, stage = 1),
+ ]
+ );
+ assert_eq!(
+ first(cache.all::<compile::Assemble>()),
+ &[
+ compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } },
+ compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } },
+ compile::Assemble { target_compiler: Compiler { host: b, stage: 1 } },
+ ]
+ );
+ assert_eq!(
+ first(cache.all::<tool::Rustdoc>()),
+ &[
+ tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } },
+ tool::Rustdoc { compiler: Compiler { host: b, stage: 1 } },
+ ],
+ );
+ assert_eq!(
+ first(cache.all::<compile::Rustc>()),
+ &[rustc!(A => A, stage = 0), rustc!(A => B, stage = 0),]
+ );
+ }
+
+ #[test]
+ fn doc_default() {
+ let mut config = configure("doc", &["A"], &["A"]);
+ config.compiler_docs = true;
+ config.cmd = Subcommand::Doc { paths: Vec::new(), open: false };
+ let mut cache = run_build(&[], config);
+ let a = TargetSelection::from_user("A");
+
+ // error_index_generator uses stage 0 to share rustdoc artifacts with the
+ // rustdoc tool.
+ assert_eq!(first(cache.all::<doc::ErrorIndex>()), &[doc::ErrorIndex { target: a },]);
+ assert_eq!(
+ first(cache.all::<tool::ErrorIndex>()),
+ &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 0 } }]
+ );
+ // docs should be built with the beta compiler, not with the stage0 artifacts.
+ // recall that rustdoc is off-by-one: `stage` is the compiler rustdoc is _linked_ to,
+ // not the one it was built by.
+ assert_eq!(
+ first(cache.all::<tool::Rustdoc>()),
+ &[tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } },]
+ );
+ }
+}
+
+mod dist {
+ use super::{first, run_build, Config};
+ use crate::builder::*;
+ use pretty_assertions::assert_eq;
+
+ fn configure(host: &[&str], target: &[&str]) -> Config {
+ Config { stage: 2, ..super::configure("dist", host, target) }
+ }
+
+ #[test]
+ fn dist_baseline() {
+ let mut cache = run_build(&[], configure(&["A"], &["A"]));
+
+ let a = TargetSelection::from_user("A");
+
+ assert_eq!(first(cache.all::<dist::Docs>()), &[dist::Docs { host: a },]);
+ assert_eq!(first(cache.all::<dist::Mingw>()), &[dist::Mingw { host: a },]);
+ assert_eq!(
+ first(cache.all::<dist::Rustc>()),
+ &[dist::Rustc { compiler: Compiler { host: a, stage: 2 } },]
+ );
+ assert_eq!(
+ first(cache.all::<dist::Std>()),
+ &[dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a },]
+ );
+ assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]);
+ // Make sure rustdoc is only built once.
+ assert_eq!(
+ first(cache.all::<tool::Rustdoc>()),
+ &[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },]
+ );
+ }
+
+ #[test]
+ fn dist_with_targets() {
+ let mut cache = run_build(&[], configure(&["A"], &["A", "B"]));
+
+ let a = TargetSelection::from_user("A");
+ let b = TargetSelection::from_user("B");
+
+ assert_eq!(
+ first(cache.all::<dist::Docs>()),
+ &[dist::Docs { host: a }, dist::Docs { host: b },]
+ );
+ assert_eq!(
+ first(cache.all::<dist::Mingw>()),
+ &[dist::Mingw { host: a }, dist::Mingw { host: b },]
+ );
+ assert_eq!(
+ first(cache.all::<dist::Rustc>()),
+ &[dist::Rustc { compiler: Compiler { host: a, stage: 2 } },]
+ );
+ assert_eq!(
+ first(cache.all::<dist::Std>()),
+ &[
+ dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a },
+ dist::Std { compiler: Compiler { host: a, stage: 2 }, target: b },
+ ]
+ );
+ assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]);
+ }
+
+ #[test]
+ fn dist_with_hosts() {
+ let mut cache = run_build(&[], configure(&["A", "B"], &["A", "B"]));
+
+ let a = TargetSelection::from_user("A");
+ let b = TargetSelection::from_user("B");
+
+ assert_eq!(
+ first(cache.all::<dist::Docs>()),
+ &[dist::Docs { host: a }, dist::Docs { host: b },]
+ );
+ assert_eq!(
+ first(cache.all::<dist::Mingw>()),
+ &[dist::Mingw { host: a }, dist::Mingw { host: b },]
+ );
+ assert_eq!(
+ first(cache.all::<dist::Rustc>()),
+ &[
+ dist::Rustc { compiler: Compiler { host: a, stage: 2 } },
+ dist::Rustc { compiler: Compiler { host: b, stage: 2 } },
+ ]
+ );
+ assert_eq!(
+ first(cache.all::<dist::Std>()),
+ &[
+ dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a },
+ dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b },
+ ]
+ );
+ assert_eq!(
+ first(cache.all::<compile::Std>()),
+ &[
+ std!(A => A, stage = 0),
+ std!(A => A, stage = 1),
+ std!(A => A, stage = 2),
+ std!(A => B, stage = 1),
+ std!(A => B, stage = 2),
+ ],
+ );
+ assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]);
+ }
+
+ #[test]
+ fn dist_only_cross_host() {
+ let b = TargetSelection::from_user("B");
+ let mut config = configure(&["A", "B"], &["A", "B"]);
+ config.docs = false;
+ config.extended = true;
+ config.hosts = vec![b];
+ let mut cache = run_build(&[], config);
+
+ assert_eq!(
+ first(cache.all::<dist::Rustc>()),
+ &[dist::Rustc { compiler: Compiler { host: b, stage: 2 } },]
+ );
+ assert_eq!(
+ first(cache.all::<compile::Rustc>()),
+ &[rustc!(A => A, stage = 0), rustc!(A => B, stage = 1),]
+ );
+ }
+
+ #[test]
+ fn dist_with_targets_and_hosts() {
+ let mut cache = run_build(&[], configure(&["A", "B"], &["A", "B", "C"]));
+
+ let a = TargetSelection::from_user("A");
+ let b = TargetSelection::from_user("B");
+ let c = TargetSelection::from_user("C");
+
+ assert_eq!(
+ first(cache.all::<dist::Docs>()),
+ &[dist::Docs { host: a }, dist::Docs { host: b }, dist::Docs { host: c },]
+ );
+ assert_eq!(
+ first(cache.all::<dist::Mingw>()),
+ &[dist::Mingw { host: a }, dist::Mingw { host: b }, dist::Mingw { host: c },]
+ );
+ assert_eq!(
+ first(cache.all::<dist::Rustc>()),
+ &[
+ dist::Rustc { compiler: Compiler { host: a, stage: 2 } },
+ dist::Rustc { compiler: Compiler { host: b, stage: 2 } },
+ ]
+ );
+ assert_eq!(
+ first(cache.all::<dist::Std>()),
+ &[
+ dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a },
+ dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b },
+ dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c },
+ ]
+ );
+ assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]);
+ }
+
+ #[test]
+ fn dist_with_empty_host() {
+ let config = configure(&[], &["C"]);
+ let mut cache = run_build(&[], config);
+
+ let a = TargetSelection::from_user("A");
+ let c = TargetSelection::from_user("C");
+
+ assert_eq!(first(cache.all::<dist::Docs>()), &[dist::Docs { host: c },]);
+ assert_eq!(first(cache.all::<dist::Mingw>()), &[dist::Mingw { host: c },]);
+ assert_eq!(
+ first(cache.all::<dist::Std>()),
+ &[dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c },]
+ );
+ }
+
+ #[test]
+ fn dist_with_same_targets_and_hosts() {
+ let mut cache = run_build(&[], configure(&["A", "B"], &["A", "B"]));
+
+ let a = TargetSelection::from_user("A");
+ let b = TargetSelection::from_user("B");
+
+ assert_eq!(
+ first(cache.all::<dist::Docs>()),
+ &[dist::Docs { host: a }, dist::Docs { host: b },]
+ );
+ assert_eq!(
+ first(cache.all::<dist::Mingw>()),
+ &[dist::Mingw { host: a }, dist::Mingw { host: b },]
+ );
+ assert_eq!(
+ first(cache.all::<dist::Rustc>()),
+ &[
+ dist::Rustc { compiler: Compiler { host: a, stage: 2 } },
+ dist::Rustc { compiler: Compiler { host: b, stage: 2 } },
+ ]
+ );
+ assert_eq!(
+ first(cache.all::<dist::Std>()),
+ &[
+ dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a },
+ dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b },
+ ]
+ );
+ assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]);
+ assert_eq!(
+ first(cache.all::<compile::Std>()),
+ &[
+ std!(A => A, stage = 0),
+ std!(A => A, stage = 1),
+ std!(A => A, stage = 2),
+ std!(A => B, stage = 1),
+ std!(A => B, stage = 2),
+ ]
+ );
+ assert_eq!(
+ first(cache.all::<compile::Assemble>()),
+ &[
+ compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } },
+ compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } },
+ compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } },
+ compile::Assemble { target_compiler: Compiler { host: b, stage: 2 } },
+ ]
+ );
+ }
+
+ #[test]
+ fn build_all() {
+ let build = Build::new(configure(&["A", "B"], &["A", "B", "C"]));
+ let mut builder = Builder::new(&build);
+ builder.run_step_descriptions(
+ &Builder::get_step_descriptions(Kind::Build),
+ &["compiler/rustc".into(), "library".into()],
+ );
+
+ assert_eq!(
+ first(builder.cache.all::<compile::Std>()),
+ &[
+ std!(A => A, stage = 0),
+ std!(A => A, stage = 1),
+ std!(A => A, stage = 2),
+ std!(A => B, stage = 1),
+ std!(A => B, stage = 2),
+ std!(A => C, stage = 2),
+ ]
+ );
+ assert_eq!(builder.cache.all::<compile::Assemble>().len(), 5);
+ assert_eq!(
+ first(builder.cache.all::<compile::Rustc>()),
+ &[
+ rustc!(A => A, stage = 0),
+ rustc!(A => A, stage = 1),
+ rustc!(A => A, stage = 2),
+ rustc!(A => B, stage = 1),
+ rustc!(A => B, stage = 2),
+ ]
+ );
+ }
+
+ #[test]
+ fn build_with_empty_host() {
+ let config = configure(&[], &["C"]);
+ let build = Build::new(config);
+ let mut builder = Builder::new(&build);
+ builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]);
+
+ let a = TargetSelection::from_user("A");
+
+ assert_eq!(
+ first(builder.cache.all::<compile::Std>()),
+ &[std!(A => A, stage = 0), std!(A => A, stage = 1), std!(A => C, stage = 2),]
+ );
+ assert_eq!(
+ first(builder.cache.all::<compile::Assemble>()),
+ &[
+ compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } },
+ compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } },
+ compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } },
+ ]
+ );
+ assert_eq!(
+ first(builder.cache.all::<compile::Rustc>()),
+ &[rustc!(A => A, stage = 0), rustc!(A => A, stage = 1),]
+ );
+ }
+
+ #[test]
+ fn test_with_no_doc_stage0() {
+ let mut config = configure(&["A"], &["A"]);
+ config.stage = 0;
+ config.cmd = Subcommand::Test {
+ paths: vec!["library/std".into()],
+ skip: vec![],
+ test_args: vec![],
+ rustc_args: vec![],
+ fail_fast: true,
+ doc_tests: DocTests::No,
+ bless: false,
+ force_rerun: false,
+ compare_mode: None,
+ rustfix_coverage: false,
+ pass: None,
+ run: None,
+ };
+
+ let build = Build::new(config);
+ let mut builder = Builder::new(&build);
+
+ let host = TargetSelection::from_user("A");
+
+ builder.run_step_descriptions(
+ &[StepDescription::from::<test::Crate>(Kind::Test)],
+ &["library/std".into()],
+ );
+
+ // Ensure we don't build any compiler artifacts.
+ assert!(!builder.cache.contains::<compile::Rustc>());
+ assert_eq!(
+ first(builder.cache.all::<test::Crate>()),
+ &[test::Crate {
+ compiler: Compiler { host, stage: 0 },
+ target: host,
+ mode: Mode::Std,
+ test_kind: test::TestKind::Test,
+ crates: vec![INTERNER.intern_str("std")],
+ },]
+ );
+ }
+
+ #[test]
+ fn doc_ci() {
+ let mut config = configure(&["A"], &["A"]);
+ config.compiler_docs = true;
+ config.cmd = Subcommand::Doc { paths: Vec::new(), open: false };
+ let build = Build::new(config);
+ let mut builder = Builder::new(&build);
+ builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Doc), &[]);
+ let a = TargetSelection::from_user("A");
+
+ // error_index_generator uses stage 1 to share rustdoc artifacts with the
+ // rustdoc tool.
+ assert_eq!(
+ first(builder.cache.all::<doc::ErrorIndex>()),
+ &[doc::ErrorIndex { target: a },]
+ );
+ assert_eq!(
+ first(builder.cache.all::<tool::ErrorIndex>()),
+ &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 1 } }]
+ );
+ // This is actually stage 1, but Rustdoc::run swaps out the compiler with
+ // stage minus 1 if --stage is not 0. Very confusing!
+ assert_eq!(
+ first(builder.cache.all::<tool::Rustdoc>()),
+ &[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },]
+ );
+ }
+
+ #[test]
+ fn test_docs() {
+ // Behavior of `x.py test` doing various documentation tests.
+ let mut config = configure(&["A"], &["A"]);
+ config.cmd = Subcommand::Test {
+ paths: vec![],
+ skip: vec![],
+ test_args: vec![],
+ rustc_args: vec![],
+ fail_fast: true,
+ doc_tests: DocTests::Yes,
+ bless: false,
+ force_rerun: false,
+ compare_mode: None,
+ rustfix_coverage: false,
+ pass: None,
+ run: None,
+ };
+ // Make sure rustfmt binary not being found isn't an error.
+ config.channel = "beta".to_string();
+ let build = Build::new(config);
+ let mut builder = Builder::new(&build);
+
+ builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Test), &[]);
+ let a = TargetSelection::from_user("A");
+
+ // error_index_generator uses stage 1 to share rustdoc artifacts with the
+ // rustdoc tool.
+ assert_eq!(
+ first(builder.cache.all::<doc::ErrorIndex>()),
+ &[doc::ErrorIndex { target: a },]
+ );
+ assert_eq!(
+ first(builder.cache.all::<tool::ErrorIndex>()),
+ &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 1 } }]
+ );
+ // Unfortunately rustdoc is built twice. Once from stage1 for compiletest
+ // (and other things), and once from stage0 for std crates. Ideally it
+ // would only be built once. If someone wants to fix this, it might be
+ // worth investigating if it would be possible to test std from stage1.
+ // Note that the stages here are +1 than what they actually are because
+ // Rustdoc::run swaps out the compiler with stage minus 1 if --stage is
+ // not 0.
+ //
+ // The stage 0 copy is the one downloaded for bootstrapping. It is
+ // (currently) needed to run "cargo test" on the linkchecker, and
+ // should be relatively "free".
+ assert_eq!(
+ first(builder.cache.all::<tool::Rustdoc>()),
+ &[
+ tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } },
+ tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } },
+ tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },
+ ]
+ );
+ }
+}
diff --git a/src/bootstrap/cache.rs b/src/bootstrap/cache.rs
new file mode 100644
index 000000000..be5c9bb07
--- /dev/null
+++ b/src/bootstrap/cache.rs
@@ -0,0 +1,273 @@
+use std::any::{Any, TypeId};
+use std::borrow::Borrow;
+use std::cell::RefCell;
+use std::cmp::{Ord, Ordering, PartialOrd};
+use std::collections::HashMap;
+use std::convert::AsRef;
+use std::fmt;
+use std::hash::{Hash, Hasher};
+use std::marker::PhantomData;
+use std::mem;
+use std::ops::Deref;
+use std::path::PathBuf;
+use std::sync::Mutex;
+
+// FIXME: replace with std::lazy after it gets stabilized and reaches beta
+use once_cell::sync::Lazy;
+
+use crate::builder::Step;
+
+pub struct Interned<T>(usize, PhantomData<*const T>);
+
+impl<T: Internable + Default> Default for Interned<T> {
+ fn default() -> Self {
+ T::default().intern()
+ }
+}
+
+impl<T> Copy for Interned<T> {}
+impl<T> Clone for Interned<T> {
+ fn clone(&self) -> Interned<T> {
+ *self
+ }
+}
+
+impl<T> PartialEq for Interned<T> {
+ fn eq(&self, other: &Self) -> bool {
+ self.0 == other.0
+ }
+}
+impl<T> Eq for Interned<T> {}
+
+impl PartialEq<str> for Interned<String> {
+ fn eq(&self, other: &str) -> bool {
+ *self == other
+ }
+}
+impl<'a> PartialEq<&'a str> for Interned<String> {
+ fn eq(&self, other: &&str) -> bool {
+ **self == **other
+ }
+}
+impl<'a, T> PartialEq<&'a Interned<T>> for Interned<T> {
+ fn eq(&self, other: &&Self) -> bool {
+ self.0 == other.0
+ }
+}
+impl<'a, T> PartialEq<Interned<T>> for &'a Interned<T> {
+ fn eq(&self, other: &Interned<T>) -> bool {
+ self.0 == other.0
+ }
+}
+
+unsafe impl<T> Send for Interned<T> {}
+unsafe impl<T> Sync for Interned<T> {}
+
+impl fmt::Display for Interned<String> {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ let s: &str = &*self;
+ f.write_str(s)
+ }
+}
+
+impl<T, U: ?Sized + fmt::Debug> fmt::Debug for Interned<T>
+where
+ Self: Deref<Target = U>,
+{
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ let s: &U = &*self;
+ f.write_fmt(format_args!("{:?}", s))
+ }
+}
+
+impl<T: Internable + Hash> Hash for Interned<T> {
+ fn hash<H: Hasher>(&self, state: &mut H) {
+ let l = T::intern_cache().lock().unwrap();
+ l.get(*self).hash(state)
+ }
+}
+
+impl<T: Internable + Deref> Deref for Interned<T> {
+ type Target = T::Target;
+ fn deref(&self) -> &'static Self::Target {
+ let l = T::intern_cache().lock().unwrap();
+ unsafe { mem::transmute::<&Self::Target, &'static Self::Target>(l.get(*self)) }
+ }
+}
+
+impl<T: Internable + AsRef<U>, U: ?Sized> AsRef<U> for Interned<T> {
+ fn as_ref(&self) -> &'static U {
+ let l = T::intern_cache().lock().unwrap();
+ unsafe { mem::transmute::<&U, &'static U>(l.get(*self).as_ref()) }
+ }
+}
+
+impl<T: Internable + PartialOrd> PartialOrd for Interned<T> {
+ fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+ let l = T::intern_cache().lock().unwrap();
+ l.get(*self).partial_cmp(l.get(*other))
+ }
+}
+
+impl<T: Internable + Ord> Ord for Interned<T> {
+ fn cmp(&self, other: &Self) -> Ordering {
+ let l = T::intern_cache().lock().unwrap();
+ l.get(*self).cmp(l.get(*other))
+ }
+}
+
+struct TyIntern<T: Clone + Eq> {
+ items: Vec<T>,
+ set: HashMap<T, Interned<T>>,
+}
+
+impl<T: Hash + Clone + Eq> Default for TyIntern<T> {
+ fn default() -> Self {
+ TyIntern { items: Vec::new(), set: Default::default() }
+ }
+}
+
+impl<T: Hash + Clone + Eq> TyIntern<T> {
+ fn intern_borrow<B>(&mut self, item: &B) -> Interned<T>
+ where
+ B: Eq + Hash + ToOwned<Owned = T> + ?Sized,
+ T: Borrow<B>,
+ {
+ if let Some(i) = self.set.get(&item) {
+ return *i;
+ }
+ let item = item.to_owned();
+ let interned = Interned(self.items.len(), PhantomData::<*const T>);
+ self.set.insert(item.clone(), interned);
+ self.items.push(item);
+ interned
+ }
+
+ fn intern(&mut self, item: T) -> Interned<T> {
+ if let Some(i) = self.set.get(&item) {
+ return *i;
+ }
+ let interned = Interned(self.items.len(), PhantomData::<*const T>);
+ self.set.insert(item.clone(), interned);
+ self.items.push(item);
+ interned
+ }
+
+ fn get(&self, i: Interned<T>) -> &T {
+ &self.items[i.0]
+ }
+}
+
+#[derive(Default)]
+pub struct Interner {
+ strs: Mutex<TyIntern<String>>,
+ paths: Mutex<TyIntern<PathBuf>>,
+ lists: Mutex<TyIntern<Vec<String>>>,
+}
+
+trait Internable: Clone + Eq + Hash + 'static {
+ fn intern_cache() -> &'static Mutex<TyIntern<Self>>;
+
+ fn intern(self) -> Interned<Self> {
+ Self::intern_cache().lock().unwrap().intern(self)
+ }
+}
+
+impl Internable for String {
+ fn intern_cache() -> &'static Mutex<TyIntern<Self>> {
+ &INTERNER.strs
+ }
+}
+
+impl Internable for PathBuf {
+ fn intern_cache() -> &'static Mutex<TyIntern<Self>> {
+ &INTERNER.paths
+ }
+}
+
+impl Internable for Vec<String> {
+ fn intern_cache() -> &'static Mutex<TyIntern<Self>> {
+ &INTERNER.lists
+ }
+}
+
+impl Interner {
+ pub fn intern_str(&self, s: &str) -> Interned<String> {
+ self.strs.lock().unwrap().intern_borrow(s)
+ }
+ pub fn intern_string(&self, s: String) -> Interned<String> {
+ self.strs.lock().unwrap().intern(s)
+ }
+
+ pub fn intern_path(&self, s: PathBuf) -> Interned<PathBuf> {
+ self.paths.lock().unwrap().intern(s)
+ }
+
+ pub fn intern_list(&self, v: Vec<String>) -> Interned<Vec<String>> {
+ self.lists.lock().unwrap().intern(v)
+ }
+}
+
+pub static INTERNER: Lazy<Interner> = Lazy::new(Interner::default);
+
+/// This is essentially a `HashMap` which allows storing any type in its input and
+/// any type in its output. It is a write-once cache; values are never evicted,
+/// which means that references to the value can safely be returned from the
+/// `get()` method.
+#[derive(Debug)]
+pub struct Cache(
+ RefCell<
+ HashMap<
+ TypeId,
+ Box<dyn Any>, // actually a HashMap<Step, Interned<Step::Output>>
+ >,
+ >,
+);
+
+impl Cache {
+ pub fn new() -> Cache {
+ Cache(RefCell::new(HashMap::new()))
+ }
+
+ pub fn put<S: Step>(&self, step: S, value: S::Output) {
+ let mut cache = self.0.borrow_mut();
+ let type_id = TypeId::of::<S>();
+ let stepcache = cache
+ .entry(type_id)
+ .or_insert_with(|| Box::new(HashMap::<S, S::Output>::new()))
+ .downcast_mut::<HashMap<S, S::Output>>()
+ .expect("invalid type mapped");
+ assert!(!stepcache.contains_key(&step), "processing {:?} a second time", step);
+ stepcache.insert(step, value);
+ }
+
+ pub fn get<S: Step>(&self, step: &S) -> Option<S::Output> {
+ let mut cache = self.0.borrow_mut();
+ let type_id = TypeId::of::<S>();
+ let stepcache = cache
+ .entry(type_id)
+ .or_insert_with(|| Box::new(HashMap::<S, S::Output>::new()))
+ .downcast_mut::<HashMap<S, S::Output>>()
+ .expect("invalid type mapped");
+ stepcache.get(step).cloned()
+ }
+}
+
+#[cfg(test)]
+impl Cache {
+ pub fn all<S: Ord + Clone + Step>(&mut self) -> Vec<(S, S::Output)> {
+ let cache = self.0.get_mut();
+ let type_id = TypeId::of::<S>();
+ let mut v = cache
+ .remove(&type_id)
+ .map(|b| b.downcast::<HashMap<S, S::Output>>().expect("correct type"))
+ .map(|m| m.into_iter().collect::<Vec<_>>())
+ .unwrap_or_default();
+ v.sort_by_key(|(s, _)| s.clone());
+ v
+ }
+
+ pub fn contains<S: Step>(&self) -> bool {
+ self.0.borrow().contains_key(&TypeId::of::<S>())
+ }
+}
diff --git a/src/bootstrap/cc_detect.rs b/src/bootstrap/cc_detect.rs
new file mode 100644
index 000000000..759a99c33
--- /dev/null
+++ b/src/bootstrap/cc_detect.rs
@@ -0,0 +1,252 @@
+//! C-compiler probing and detection.
+//!
+//! This module will fill out the `cc` and `cxx` maps of `Build` by looking for
+//! C and C++ compilers for each target configured. A compiler is found through
+//! a number of vectors (in order of precedence)
+//!
+//! 1. Configuration via `target.$target.cc` in `config.toml`.
+//! 2. Configuration via `target.$target.android-ndk` in `config.toml`, if
+//! applicable
+//! 3. Special logic to probe on OpenBSD
+//! 4. The `CC_$target` environment variable.
+//! 5. The `CC` environment variable.
+//! 6. "cc"
+//!
+//! Some of this logic is implemented here, but much of it is farmed out to the
+//! `cc` crate itself, so we end up having the same fallbacks as there.
+//! Similar logic is then used to find a C++ compiler, just some s/cc/c++/ is
+//! used.
+//!
+//! It is intended that after this module has run no C/C++ compiler will
+//! ever be probed for. Instead the compilers found here will be used for
+//! everything.
+
+use std::collections::HashSet;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+use std::{env, iter};
+
+use crate::config::{Target, TargetSelection};
+use crate::util::output;
+use crate::{Build, CLang, GitRepo};
+
+// The `cc` crate doesn't provide a way to obtain a path to the detected archiver,
+// so use some simplified logic here. First we respect the environment variable `AR`, then
+// try to infer the archiver path from the C compiler path.
+// In the future this logic should be replaced by calling into the `cc` crate.
+fn cc2ar(cc: &Path, target: TargetSelection) -> Option<PathBuf> {
+ if let Some(ar) = env::var_os(format!("AR_{}", target.triple.replace("-", "_"))) {
+ Some(PathBuf::from(ar))
+ } else if let Some(ar) = env::var_os("AR") {
+ Some(PathBuf::from(ar))
+ } else if target.contains("msvc") {
+ None
+ } else if target.contains("musl") {
+ Some(PathBuf::from("ar"))
+ } else if target.contains("openbsd") {
+ Some(PathBuf::from("ar"))
+ } else if target.contains("vxworks") {
+ Some(PathBuf::from("wr-ar"))
+ } else {
+ let parent = cc.parent().unwrap();
+ let file = cc.file_name().unwrap().to_str().unwrap();
+ for suffix in &["gcc", "cc", "clang"] {
+ if let Some(idx) = file.rfind(suffix) {
+ let mut file = file[..idx].to_owned();
+ file.push_str("ar");
+ return Some(parent.join(&file));
+ }
+ }
+ Some(parent.join(file))
+ }
+}
+
+fn new_cc_build(build: &Build, target: TargetSelection) -> cc::Build {
+ let mut cfg = cc::Build::new();
+ cfg.cargo_metadata(false)
+ .opt_level(2)
+ .warnings(false)
+ .debug(false)
+ .target(&target.triple)
+ .host(&build.build.triple);
+ match build.crt_static(target) {
+ Some(a) => {
+ cfg.static_crt(a);
+ }
+ None => {
+ if target.contains("msvc") {
+ cfg.static_crt(true);
+ }
+ if target.contains("musl") {
+ cfg.static_flag(true);
+ }
+ }
+ }
+ cfg
+}
+
+pub fn find(build: &mut Build) {
+ // For all targets we're going to need a C compiler for building some shims
+ // and such as well as for being a linker for Rust code.
+ let targets = build
+ .targets
+ .iter()
+ .chain(&build.hosts)
+ .cloned()
+ .chain(iter::once(build.build))
+ .collect::<HashSet<_>>();
+ for target in targets.into_iter() {
+ let mut cfg = new_cc_build(build, target);
+ let config = build.config.target_config.get(&target);
+ if let Some(cc) = config.and_then(|c| c.cc.as_ref()) {
+ cfg.compiler(cc);
+ } else {
+ set_compiler(&mut cfg, Language::C, target, config, build);
+ }
+
+ let compiler = cfg.get_compiler();
+ let ar = if let ar @ Some(..) = config.and_then(|c| c.ar.clone()) {
+ ar
+ } else {
+ cc2ar(compiler.path(), target)
+ };
+
+ build.cc.insert(target, compiler.clone());
+ let cflags = build.cflags(target, GitRepo::Rustc, CLang::C);
+
+ // If we use llvm-libunwind, we will need a C++ compiler as well for all targets
+ // We'll need one anyways if the target triple is also a host triple
+ let mut cfg = new_cc_build(build, target);
+ cfg.cpp(true);
+ let cxx_configured = if let Some(cxx) = config.and_then(|c| c.cxx.as_ref()) {
+ cfg.compiler(cxx);
+ true
+ } else if build.hosts.contains(&target) || build.build == target {
+ set_compiler(&mut cfg, Language::CPlusPlus, target, config, build);
+ true
+ } else {
+ // Use an auto-detected compiler (or one configured via `CXX_target_triple` env vars).
+ cfg.try_get_compiler().is_ok()
+ };
+
+ // for VxWorks, record CXX compiler which will be used in lib.rs:linker()
+ if cxx_configured || target.contains("vxworks") {
+ let compiler = cfg.get_compiler();
+ build.cxx.insert(target, compiler);
+ }
+
+ build.verbose(&format!("CC_{} = {:?}", &target.triple, build.cc(target)));
+ build.verbose(&format!("CFLAGS_{} = {:?}", &target.triple, cflags));
+ if let Ok(cxx) = build.cxx(target) {
+ let cxxflags = build.cflags(target, GitRepo::Rustc, CLang::Cxx);
+ build.verbose(&format!("CXX_{} = {:?}", &target.triple, cxx));
+ build.verbose(&format!("CXXFLAGS_{} = {:?}", &target.triple, cxxflags));
+ }
+ if let Some(ar) = ar {
+ build.verbose(&format!("AR_{} = {:?}", &target.triple, ar));
+ build.ar.insert(target, ar);
+ }
+
+ if let Some(ranlib) = config.and_then(|c| c.ranlib.clone()) {
+ build.ranlib.insert(target, ranlib);
+ }
+ }
+}
+
+fn set_compiler(
+ cfg: &mut cc::Build,
+ compiler: Language,
+ target: TargetSelection,
+ config: Option<&Target>,
+ build: &Build,
+) {
+ match &*target.triple {
+ // When compiling for android we may have the NDK configured in the
+ // config.toml in which case we look there. Otherwise the default
+ // compiler already takes into account the triple in question.
+ t if t.contains("android") => {
+ if let Some(ndk) = config.and_then(|c| c.ndk.as_ref()) {
+ let target = target
+ .triple
+ .replace("armv7neon", "arm")
+ .replace("armv7", "arm")
+ .replace("thumbv7neon", "arm")
+ .replace("thumbv7", "arm");
+ let compiler = format!("{}-{}", target, compiler.clang());
+ cfg.compiler(ndk.join("bin").join(compiler));
+ }
+ }
+
+ // The default gcc version from OpenBSD may be too old, try using egcc,
+ // which is a gcc version from ports, if this is the case.
+ t if t.contains("openbsd") => {
+ let c = cfg.get_compiler();
+ let gnu_compiler = compiler.gcc();
+ if !c.path().ends_with(gnu_compiler) {
+ return;
+ }
+
+ let output = output(c.to_command().arg("--version"));
+ let i = match output.find(" 4.") {
+ Some(i) => i,
+ None => return,
+ };
+ match output[i + 3..].chars().next().unwrap() {
+ '0'..='6' => {}
+ _ => return,
+ }
+ let alternative = format!("e{}", gnu_compiler);
+ if Command::new(&alternative).output().is_ok() {
+ cfg.compiler(alternative);
+ }
+ }
+
+ "mips-unknown-linux-musl" => {
+ if cfg.get_compiler().path().to_str() == Some("gcc") {
+ cfg.compiler("mips-linux-musl-gcc");
+ }
+ }
+ "mipsel-unknown-linux-musl" => {
+ if cfg.get_compiler().path().to_str() == Some("gcc") {
+ cfg.compiler("mipsel-linux-musl-gcc");
+ }
+ }
+
+ t if t.contains("musl") => {
+ if let Some(root) = build.musl_root(target) {
+ let guess = root.join("bin/musl-gcc");
+ if guess.exists() {
+ cfg.compiler(guess);
+ }
+ }
+ }
+
+ _ => {}
+ }
+}
+
+/// The target programming language for a native compiler.
+enum Language {
+ /// The compiler is targeting C.
+ C,
+ /// The compiler is targeting C++.
+ CPlusPlus,
+}
+
+impl Language {
+ /// Obtains the name of a compiler in the GCC collection.
+ fn gcc(self) -> &'static str {
+ match self {
+ Language::C => "gcc",
+ Language::CPlusPlus => "g++",
+ }
+ }
+
+ /// Obtains the name of a compiler in the clang suite.
+ fn clang(self) -> &'static str {
+ match self {
+ Language::C => "clang",
+ Language::CPlusPlus => "clang++",
+ }
+ }
+}
diff --git a/src/bootstrap/channel.rs b/src/bootstrap/channel.rs
new file mode 100644
index 000000000..1932a0017
--- /dev/null
+++ b/src/bootstrap/channel.rs
@@ -0,0 +1,105 @@
+//! Build configuration for Rust's release channels.
+//!
+//! Implements the stable/beta/nightly channel distinctions by setting various
+//! flags like the `unstable_features`, calculating variables like `release` and
+//! `package_vers`, and otherwise indicating to the compiler what it should
+//! print out as part of its version information.
+
+use std::path::Path;
+use std::process::Command;
+
+use crate::util::output;
+use crate::Build;
+
+pub enum GitInfo {
+ /// This is not a git repository.
+ Absent,
+ /// This is a git repository.
+ /// If the info should be used (`ignore_git` is false), this will be
+ /// `Some`, otherwise it will be `None`.
+ Present(Option<Info>),
+}
+
+pub struct Info {
+ commit_date: String,
+ sha: String,
+ short_sha: String,
+}
+
+impl GitInfo {
+ pub fn new(ignore_git: bool, dir: &Path) -> GitInfo {
+ // See if this even begins to look like a git dir
+ if !dir.join(".git").exists() {
+ return GitInfo::Absent;
+ }
+
+ // Make sure git commands work
+ match Command::new("git").arg("rev-parse").current_dir(dir).output() {
+ Ok(ref out) if out.status.success() => {}
+ _ => return GitInfo::Absent,
+ }
+
+ // If we're ignoring the git info, we don't actually need to collect it, just make sure this
+ // was a git repo in the first place.
+ if ignore_git {
+ return GitInfo::Present(None);
+ }
+
+ // Ok, let's scrape some info
+ let ver_date = output(
+ Command::new("git")
+ .current_dir(dir)
+ .arg("log")
+ .arg("-1")
+ .arg("--date=short")
+ .arg("--pretty=format:%cd"),
+ );
+ let ver_hash = output(Command::new("git").current_dir(dir).arg("rev-parse").arg("HEAD"));
+ let short_ver_hash = output(
+ Command::new("git").current_dir(dir).arg("rev-parse").arg("--short=9").arg("HEAD"),
+ );
+ GitInfo::Present(Some(Info {
+ commit_date: ver_date.trim().to_string(),
+ sha: ver_hash.trim().to_string(),
+ short_sha: short_ver_hash.trim().to_string(),
+ }))
+ }
+
+ fn info(&self) -> Option<&Info> {
+ match self {
+ GitInfo::Present(info) => info.as_ref(),
+ GitInfo::Absent => None,
+ }
+ }
+
+ pub fn sha(&self) -> Option<&str> {
+ self.info().map(|s| &s.sha[..])
+ }
+
+ pub fn sha_short(&self) -> Option<&str> {
+ self.info().map(|s| &s.short_sha[..])
+ }
+
+ pub fn commit_date(&self) -> Option<&str> {
+ self.info().map(|s| &s.commit_date[..])
+ }
+
+ pub fn version(&self, build: &Build, num: &str) -> String {
+ let mut version = build.release(num);
+ if let Some(ref inner) = self.info() {
+ version.push_str(" (");
+ version.push_str(&inner.short_sha);
+ version.push(' ');
+ version.push_str(&inner.commit_date);
+ version.push(')');
+ }
+ version
+ }
+
+ pub fn is_git(&self) -> bool {
+ match self {
+ GitInfo::Absent => false,
+ GitInfo::Present(_) => true,
+ }
+ }
+}
diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs
new file mode 100644
index 000000000..4e1e8ef9d
--- /dev/null
+++ b/src/bootstrap/check.rs
@@ -0,0 +1,498 @@
+//! Implementation of compiling the compiler and standard library, in "check"-based modes.
+
+use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
+use crate::cache::Interned;
+use crate::compile::{add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo};
+use crate::config::TargetSelection;
+use crate::tool::{prepare_tool_cargo, SourceType};
+use crate::INTERNER;
+use crate::{Compiler, Mode, Subcommand};
+use std::path::{Path, PathBuf};
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct Std {
+ pub target: TargetSelection,
+}
+
+/// Returns args for the subcommand itself (not for cargo)
+fn args(builder: &Builder<'_>) -> Vec<String> {
+ fn strings<'a>(arr: &'a [&str]) -> impl Iterator<Item = String> + 'a {
+ arr.iter().copied().map(String::from)
+ }
+
+ if let Subcommand::Clippy {
+ fix,
+ clippy_lint_allow,
+ clippy_lint_deny,
+ clippy_lint_warn,
+ clippy_lint_forbid,
+ ..
+ } = &builder.config.cmd
+ {
+ // disable the most spammy clippy lints
+ let ignored_lints = vec![
+ "many_single_char_names", // there are a lot in stdarch
+ "collapsible_if",
+ "type_complexity",
+ "missing_safety_doc", // almost 3K warnings
+ "too_many_arguments",
+ "needless_lifetimes", // people want to keep the lifetimes
+ "wrong_self_convention",
+ ];
+ let mut args = vec![];
+ if *fix {
+ #[rustfmt::skip]
+ args.extend(strings(&[
+ "--fix", "-Zunstable-options",
+ // FIXME: currently, `--fix` gives an error while checking tests for libtest,
+ // possibly because libtest is not yet built in the sysroot.
+ // As a workaround, avoid checking tests and benches when passed --fix.
+ "--lib", "--bins", "--examples",
+ ]));
+ }
+ args.extend(strings(&["--", "--cap-lints", "warn"]));
+ args.extend(ignored_lints.iter().map(|lint| format!("-Aclippy::{}", lint)));
+ let mut clippy_lint_levels: Vec<String> = Vec::new();
+ clippy_lint_allow.iter().for_each(|v| clippy_lint_levels.push(format!("-A{}", v)));
+ clippy_lint_deny.iter().for_each(|v| clippy_lint_levels.push(format!("-D{}", v)));
+ clippy_lint_warn.iter().for_each(|v| clippy_lint_levels.push(format!("-W{}", v)));
+ clippy_lint_forbid.iter().for_each(|v| clippy_lint_levels.push(format!("-F{}", v)));
+ args.extend(clippy_lint_levels);
+ args
+ } else {
+ vec![]
+ }
+}
+
+fn cargo_subcommand(kind: Kind) -> &'static str {
+ match kind {
+ Kind::Check => "check",
+ Kind::Clippy => "clippy",
+ Kind::Fix => "fix",
+ _ => unreachable!(),
+ }
+}
+
+impl Step for Std {
+ type Output = ();
+ const DEFAULT: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.all_krates("test").path("library")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Std { target: run.target });
+ }
+
+ fn run(self, builder: &Builder<'_>) {
+ builder.update_submodule(&Path::new("library").join("stdarch"));
+
+ let target = self.target;
+ let compiler = builder.compiler(builder.top_stage, builder.config.build);
+
+ let mut cargo = builder.cargo(
+ compiler,
+ Mode::Std,
+ SourceType::InTree,
+ target,
+ cargo_subcommand(builder.kind),
+ );
+ std_cargo(builder, target, compiler.stage, &mut cargo);
+
+ builder.info(&format!(
+ "Checking stage{} std artifacts ({} -> {})",
+ builder.top_stage, &compiler.host, target
+ ));
+ run_cargo(
+ builder,
+ cargo,
+ args(builder),
+ &libstd_stamp(builder, compiler, target),
+ vec![],
+ true,
+ );
+
+ // We skip populating the sysroot in non-zero stage because that'll lead
+ // to rlib/rmeta conflicts if std gets built during this session.
+ if compiler.stage == 0 {
+ let libdir = builder.sysroot_libdir(compiler, target);
+ let hostdir = builder.sysroot_libdir(compiler, compiler.host);
+ add_to_sysroot(&builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target));
+ }
+
+ // don't run on std twice with x.py clippy
+ if builder.kind == Kind::Clippy {
+ return;
+ }
+
+ // Then run cargo again, once we've put the rmeta files for the library
+ // crates into the sysroot. This is needed because e.g., core's tests
+ // depend on `libtest` -- Cargo presumes it will exist, but it doesn't
+ // since we initialize with an empty sysroot.
+ //
+ // Currently only the "libtest" tree of crates does this.
+ let mut cargo = builder.cargo(
+ compiler,
+ Mode::Std,
+ SourceType::InTree,
+ target,
+ cargo_subcommand(builder.kind),
+ );
+
+ // If we're not in stage 0, tests and examples will fail to compile
+ // from `core` definitions being loaded from two different `libcore`
+ // .rmeta and .rlib files.
+ if compiler.stage == 0 {
+ cargo.arg("--all-targets");
+ }
+
+ std_cargo(builder, target, compiler.stage, &mut cargo);
+
+ // Explicitly pass -p for all dependencies krates -- this will force cargo
+ // to also check the tests/benches/examples for these crates, rather
+ // than just the leaf crate.
+ for krate in builder.in_tree_crates("test", Some(target)) {
+ cargo.arg("-p").arg(krate.name);
+ }
+
+ builder.info(&format!(
+ "Checking stage{} std test/bench/example targets ({} -> {})",
+ builder.top_stage, &compiler.host, target
+ ));
+ run_cargo(
+ builder,
+ cargo,
+ args(builder),
+ &libstd_test_stamp(builder, compiler, target),
+ vec![],
+ true,
+ );
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct Rustc {
+ pub target: TargetSelection,
+}
+
+impl Step for Rustc {
+ type Output = ();
+ const ONLY_HOSTS: bool = true;
+ const DEFAULT: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.all_krates("rustc-main").path("compiler")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Rustc { target: run.target });
+ }
+
+ /// Builds the compiler.
+ ///
+ /// This will build the compiler for a particular stage of the build using
+ /// the `compiler` targeting the `target` architecture. The artifacts
+ /// created will also be linked into the sysroot directory.
+ fn run(self, builder: &Builder<'_>) {
+ let compiler = builder.compiler(builder.top_stage, builder.config.build);
+ let target = self.target;
+
+ if compiler.stage != 0 {
+ // If we're not in stage 0, then we won't have a std from the beta
+ // compiler around. That means we need to make sure there's one in
+ // the sysroot for the compiler to find. Otherwise, we're going to
+ // fail when building crates that need to generate code (e.g., build
+ // scripts and their dependencies).
+ builder.ensure(crate::compile::Std::new(compiler, compiler.host));
+ builder.ensure(crate::compile::Std::new(compiler, target));
+ } else {
+ builder.ensure(Std { target });
+ }
+
+ let mut cargo = builder.cargo(
+ compiler,
+ Mode::Rustc,
+ SourceType::InTree,
+ target,
+ cargo_subcommand(builder.kind),
+ );
+ rustc_cargo(builder, &mut cargo, target);
+
+ // For ./x.py clippy, don't run with --all-targets because
+ // linting tests and benchmarks can produce very noisy results
+ if builder.kind != Kind::Clippy {
+ cargo.arg("--all-targets");
+ }
+
+ // Explicitly pass -p for all compiler krates -- this will force cargo
+ // to also check the tests/benches/examples for these crates, rather
+ // than just the leaf crate.
+ for krate in builder.in_tree_crates("rustc-main", Some(target)) {
+ cargo.arg("-p").arg(krate.name);
+ }
+
+ builder.info(&format!(
+ "Checking stage{} compiler artifacts ({} -> {})",
+ builder.top_stage, &compiler.host, target
+ ));
+ run_cargo(
+ builder,
+ cargo,
+ args(builder),
+ &librustc_stamp(builder, compiler, target),
+ vec![],
+ true,
+ );
+
+ let libdir = builder.sysroot_libdir(compiler, target);
+ let hostdir = builder.sysroot_libdir(compiler, compiler.host);
+ add_to_sysroot(&builder, &libdir, &hostdir, &librustc_stamp(builder, compiler, target));
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct CodegenBackend {
+ pub target: TargetSelection,
+ pub backend: Interned<String>,
+}
+
+impl Step for CodegenBackend {
+ type Output = ();
+ const ONLY_HOSTS: bool = true;
+ const DEFAULT: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.paths(&["compiler/rustc_codegen_cranelift", "compiler/rustc_codegen_gcc"])
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ for &backend in &[INTERNER.intern_str("cranelift"), INTERNER.intern_str("gcc")] {
+ run.builder.ensure(CodegenBackend { target: run.target, backend });
+ }
+ }
+
+ fn run(self, builder: &Builder<'_>) {
+ let compiler = builder.compiler(builder.top_stage, builder.config.build);
+ let target = self.target;
+ let backend = self.backend;
+
+ builder.ensure(Rustc { target });
+
+ let mut cargo = builder.cargo(
+ compiler,
+ Mode::Codegen,
+ SourceType::InTree,
+ target,
+ cargo_subcommand(builder.kind),
+ );
+ cargo
+ .arg("--manifest-path")
+ .arg(builder.src.join(format!("compiler/rustc_codegen_{}/Cargo.toml", backend)));
+ rustc_cargo_env(builder, &mut cargo, target);
+
+ builder.info(&format!(
+ "Checking stage{} {} artifacts ({} -> {})",
+ builder.top_stage, backend, &compiler.host.triple, target.triple
+ ));
+
+ run_cargo(
+ builder,
+ cargo,
+ args(builder),
+ &codegen_backend_stamp(builder, compiler, target, backend),
+ vec![],
+ true,
+ );
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct RustAnalyzer {
+ pub target: TargetSelection,
+}
+
+impl Step for RustAnalyzer {
+ type Output = ();
+ const ONLY_HOSTS: bool = true;
+ const DEFAULT: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/tools/rust-analyzer")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(RustAnalyzer { target: run.target });
+ }
+
+ fn run(self, builder: &Builder<'_>) {
+ let compiler = builder.compiler(builder.top_stage, builder.config.build);
+ let target = self.target;
+
+ builder.ensure(Std { target });
+
+ let mut cargo = prepare_tool_cargo(
+ builder,
+ compiler,
+ Mode::ToolStd,
+ target,
+ cargo_subcommand(builder.kind),
+ "src/tools/rust-analyzer",
+ SourceType::InTree,
+ &["rust-analyzer/in-rust-tree".to_owned()],
+ );
+
+ cargo.rustflag(
+ "-Zallow-features=proc_macro_internals,proc_macro_diagnostic,proc_macro_span",
+ );
+
+ // For ./x.py clippy, don't check those targets because
+ // linting tests and benchmarks can produce very noisy results
+ if builder.kind != Kind::Clippy {
+ // can't use `--all-targets` because `--examples` doesn't work well
+ cargo.arg("--bins");
+ cargo.arg("--tests");
+ cargo.arg("--benches");
+ }
+
+ builder.info(&format!(
+ "Checking stage{} {} artifacts ({} -> {})",
+ compiler.stage, "rust-analyzer", &compiler.host.triple, target.triple
+ ));
+ run_cargo(builder, cargo, args(builder), &stamp(builder, compiler, target), vec![], true);
+
+ /// Cargo's output path in a given stage, compiled by a particular
+ /// compiler for the specified target.
+ fn stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf {
+ builder.cargo_out(compiler, Mode::ToolStd, target).join(".rust-analyzer-check.stamp")
+ }
+ }
+}
+
+macro_rules! tool_check_step {
+ ($name:ident, $path:literal, $($alias:literal, )* $source_type:path $(, $default:literal )?) => {
+ #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+ pub struct $name {
+ pub target: TargetSelection,
+ }
+
+ impl Step for $name {
+ type Output = ();
+ const ONLY_HOSTS: bool = true;
+ // don't ever check out-of-tree tools by default, they'll fail when toolstate is broken
+ const DEFAULT: bool = matches!($source_type, SourceType::InTree) $( && $default )?;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.paths(&[ $path, $($alias),* ])
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure($name { target: run.target });
+ }
+
+ fn run(self, builder: &Builder<'_>) {
+ let compiler = builder.compiler(builder.top_stage, builder.config.build);
+ let target = self.target;
+
+ builder.ensure(Rustc { target });
+
+ let mut cargo = prepare_tool_cargo(
+ builder,
+ compiler,
+ Mode::ToolRustc,
+ target,
+ cargo_subcommand(builder.kind),
+ $path,
+ $source_type,
+ &[],
+ );
+
+ // For ./x.py clippy, don't run with --all-targets because
+ // linting tests and benchmarks can produce very noisy results
+ if builder.kind != Kind::Clippy {
+ cargo.arg("--all-targets");
+ }
+
+ // Enable internal lints for clippy and rustdoc
+ // NOTE: this doesn't enable lints for any other tools unless they explicitly add `#![warn(rustc::internal)]`
+ // See https://github.com/rust-lang/rust/pull/80573#issuecomment-754010776
+ cargo.rustflag("-Zunstable-options");
+
+ builder.info(&format!(
+ "Checking stage{} {} artifacts ({} -> {})",
+ builder.top_stage,
+ stringify!($name).to_lowercase(),
+ &compiler.host.triple,
+ target.triple
+ ));
+ run_cargo(
+ builder,
+ cargo,
+ args(builder),
+ &stamp(builder, compiler, target),
+ vec![],
+ true,
+ );
+
+ /// Cargo's output path in a given stage, compiled by a particular
+ /// compiler for the specified target.
+ fn stamp(
+ builder: &Builder<'_>,
+ compiler: Compiler,
+ target: TargetSelection,
+ ) -> PathBuf {
+ builder
+ .cargo_out(compiler, Mode::ToolRustc, target)
+ .join(format!(".{}-check.stamp", stringify!($name).to_lowercase()))
+ }
+ }
+ }
+ };
+}
+
+tool_check_step!(Rustdoc, "src/tools/rustdoc", "src/librustdoc", SourceType::InTree);
+// Clippy and Rustfmt are hybrids. They are external tools, but use a git subtree instead
+// of a submodule. Since the SourceType only drives the deny-warnings
+// behavior, treat it as in-tree so that any new warnings in clippy will be
+// rejected.
+tool_check_step!(Clippy, "src/tools/clippy", SourceType::InTree);
+tool_check_step!(Miri, "src/tools/miri", SourceType::Submodule);
+tool_check_step!(Rls, "src/tools/rls", SourceType::Submodule);
+tool_check_step!(Rustfmt, "src/tools/rustfmt", SourceType::InTree);
+
+tool_check_step!(Bootstrap, "src/bootstrap", SourceType::InTree, false);
+
+/// Cargo's output path for the standard library in a given stage, compiled
+/// by a particular compiler for the specified target.
+fn libstd_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf {
+ builder.cargo_out(compiler, Mode::Std, target).join(".libstd-check.stamp")
+}
+
+/// Cargo's output path for the standard library in a given stage, compiled
+/// by a particular compiler for the specified target.
+fn libstd_test_stamp(
+ builder: &Builder<'_>,
+ compiler: Compiler,
+ target: TargetSelection,
+) -> PathBuf {
+ builder.cargo_out(compiler, Mode::Std, target).join(".libstd-check-test.stamp")
+}
+
+/// Cargo's output path for librustc in a given stage, compiled by a particular
+/// compiler for the specified target.
+fn librustc_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf {
+ builder.cargo_out(compiler, Mode::Rustc, target).join(".librustc-check.stamp")
+}
+
+/// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular
+/// compiler for the specified target and backend.
+fn codegen_backend_stamp(
+ builder: &Builder<'_>,
+ compiler: Compiler,
+ target: TargetSelection,
+ backend: Interned<String>,
+) -> PathBuf {
+ builder
+ .cargo_out(compiler, Mode::Codegen, target)
+ .join(format!(".librustc_codegen_{}-check.stamp", backend))
+}
diff --git a/src/bootstrap/clean.rs b/src/bootstrap/clean.rs
new file mode 100644
index 000000000..069f3d6ac
--- /dev/null
+++ b/src/bootstrap/clean.rs
@@ -0,0 +1,118 @@
+//! Implementation of `make clean` in rustbuild.
+//!
+//! Responsible for cleaning out a build directory of all old and stale
+//! artifacts to prepare for a fresh build. Currently doesn't remove the
+//! `build/cache` directory (download cache) or the `build/$target/llvm`
+//! directory unless the `--all` flag is present.
+
+use std::fs;
+use std::io::{self, ErrorKind};
+use std::path::Path;
+
+use crate::util::t;
+use crate::Build;
+
+pub fn clean(build: &Build, all: bool) {
+ rm_rf("tmp".as_ref());
+
+ if all {
+ rm_rf(&build.out);
+ } else {
+ rm_rf(&build.out.join("tmp"));
+ rm_rf(&build.out.join("dist"));
+ rm_rf(&build.out.join("bootstrap"));
+
+ for host in &build.hosts {
+ let entries = match build.out.join(host.triple).read_dir() {
+ Ok(iter) => iter,
+ Err(_) => continue,
+ };
+
+ for entry in entries {
+ let entry = t!(entry);
+ if entry.file_name().to_str() == Some("llvm") {
+ continue;
+ }
+ let path = t!(entry.path().canonicalize());
+ rm_rf(&path);
+ }
+ }
+ }
+}
+
+fn rm_rf(path: &Path) {
+ match path.symlink_metadata() {
+ Err(e) => {
+ if e.kind() == ErrorKind::NotFound {
+ return;
+ }
+ panic!("failed to get metadata for file {}: {}", path.display(), e);
+ }
+ Ok(metadata) => {
+ if metadata.file_type().is_file() || metadata.file_type().is_symlink() {
+ do_op(path, "remove file", |p| {
+ fs::remove_file(p).or_else(|e| {
+ // Work around the fact that we cannot
+ // delete an executable while it runs on Windows.
+ #[cfg(windows)]
+ if e.kind() == std::io::ErrorKind::PermissionDenied
+ && p.file_name().and_then(std::ffi::OsStr::to_str)
+ == Some("bootstrap.exe")
+ {
+ eprintln!("warning: failed to delete '{}'.", p.display());
+ return Ok(());
+ }
+ Err(e)
+ })
+ });
+ return;
+ }
+
+ for file in t!(fs::read_dir(path)) {
+ rm_rf(&t!(file).path());
+ }
+ do_op(path, "remove dir", |p| {
+ fs::remove_dir(p).or_else(|e| {
+ // Check for dir not empty on Windows
+ // FIXME: Once `ErrorKind::DirectoryNotEmpty` is stabilized,
+ // match on `e.kind()` instead.
+ #[cfg(windows)]
+ if e.raw_os_error() == Some(145) {
+ return Ok(());
+ }
+
+ Err(e)
+ })
+ });
+ }
+ };
+}
+
+fn do_op<F>(path: &Path, desc: &str, mut f: F)
+where
+ F: FnMut(&Path) -> io::Result<()>,
+{
+ match f(path) {
+ Ok(()) => {}
+ // On windows we can't remove a readonly file, and git will often clone files as readonly.
+ // As a result, we have some special logic to remove readonly files on windows.
+ // This is also the reason that we can't use things like fs::remove_dir_all().
+ Err(ref e) if cfg!(windows) && e.kind() == ErrorKind::PermissionDenied => {
+ let m = t!(path.symlink_metadata());
+ let mut p = m.permissions();
+ p.set_readonly(false);
+ t!(fs::set_permissions(path, p));
+ f(path).unwrap_or_else(|e| {
+ // Delete symlinked directories on Windows
+ #[cfg(windows)]
+ if m.file_type().is_symlink() && path.is_dir() && fs::remove_dir(path).is_ok() {
+ return;
+ }
+ panic!("failed to {} {}: {}", desc, path.display(), e);
+ });
+ }
+ Err(e) => {
+ panic!("failed to {} {}: {}", desc, path.display(), e);
+ }
+ }
+}
diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs
new file mode 100644
index 000000000..dd2b9d593
--- /dev/null
+++ b/src/bootstrap/compile.rs
@@ -0,0 +1,1571 @@
+//! Implementation of compiling various phases of the compiler and standard
+//! library.
+//!
+//! This module contains some of the real meat in the rustbuild build system
+//! which is where Cargo is used to compile the standard library, libtest, and
+//! the compiler. This module is also responsible for assembling the sysroot as it
+//! goes along from the output of the previous stage.
+
+use std::borrow::Cow;
+use std::collections::HashSet;
+use std::env;
+use std::fs;
+use std::io::prelude::*;
+use std::io::BufReader;
+use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
+use std::str;
+
+use serde::Deserialize;
+
+use crate::builder::Cargo;
+use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
+use crate::cache::{Interned, INTERNER};
+use crate::config::{LlvmLibunwind, TargetSelection};
+use crate::dist;
+use crate::native;
+use crate::tool::SourceType;
+use crate::util::get_clang_cl_resource_dir;
+use crate::util::{exe, is_debug_info, is_dylib, output, symlink_dir, t, up_to_date};
+use crate::LLVM_TOOLS;
+use crate::{CLang, Compiler, DependencyType, GitRepo, Mode};
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct Std {
+ pub target: TargetSelection,
+ pub compiler: Compiler,
+ /// Whether to build only a subset of crates in the standard library.
+ ///
+ /// This shouldn't be used from other steps; see the comment on [`Rustc`].
+ crates: Interned<Vec<String>>,
+}
+
+impl Std {
+ pub fn new(compiler: Compiler, target: TargetSelection) -> Self {
+ Self { target, compiler, crates: Default::default() }
+ }
+}
+
+/// Return a `-p=x -p=y` string suitable for passing to a cargo invocation.
+fn build_crates_in_set(run: &RunConfig<'_>) -> Interned<Vec<String>> {
+ let mut crates = Vec::new();
+ for krate in &run.paths {
+ let path = krate.assert_single_path();
+ let crate_name = run.builder.crate_paths[&path.path];
+ crates.push(format!("-p={crate_name}"));
+ }
+ INTERNER.intern_list(crates)
+}
+
+impl Step for Std {
+ type Output = ();
+ const DEFAULT: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ // When downloading stage1, the standard library has already been copied to the sysroot, so
+ // there's no need to rebuild it.
+ let builder = run.builder;
+ run.crate_or_deps("test")
+ .path("library")
+ .lazy_default_condition(Box::new(|| !builder.download_rustc()))
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ // Normally, people will pass *just* library if they pass it.
+ // But it's possible (although strange) to pass something like `library std core`.
+ // Build all crates anyway, as if they hadn't passed the other args.
+ let has_library =
+ run.paths.iter().any(|set| set.assert_single_path().path.ends_with("library"));
+ let crates = if has_library { Default::default() } else { build_crates_in_set(&run) };
+ run.builder.ensure(Std {
+ compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()),
+ target: run.target,
+ crates,
+ });
+ }
+
+ /// Builds the standard library.
+ ///
+ /// This will build the standard library for a particular stage of the build
+ /// using the `compiler` targeting the `target` architecture. The artifacts
+ /// created will also be linked into the sysroot directory.
+ fn run(self, builder: &Builder<'_>) {
+ let target = self.target;
+ let compiler = self.compiler;
+
+ // These artifacts were already copied (in `impl Step for Sysroot`).
+ // Don't recompile them.
+ // NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler,
+ // so its artifacts can't be reused.
+ if builder.download_rustc() && compiler.stage != 0 {
+ return;
+ }
+
+ if builder.config.keep_stage.contains(&compiler.stage)
+ || builder.config.keep_stage_std.contains(&compiler.stage)
+ {
+ builder.info("Warning: Using a potentially old libstd. This may not behave well.");
+ builder.ensure(StdLink::from_std(self, compiler));
+ return;
+ }
+
+ builder.update_submodule(&Path::new("library").join("stdarch"));
+
+ // Profiler information requires LLVM's compiler-rt
+ if builder.config.profiler {
+ builder.update_submodule(&Path::new("src/llvm-project"));
+ }
+
+ let mut target_deps = builder.ensure(StartupObjects { compiler, target });
+
+ let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
+ if compiler_to_use != compiler {
+ builder.ensure(Std::new(compiler_to_use, target));
+ builder.info(&format!("Uplifting stage1 std ({} -> {})", compiler_to_use.host, target));
+
+ // Even if we're not building std this stage, the new sysroot must
+ // still contain the third party objects needed by various targets.
+ copy_third_party_objects(builder, &compiler, target);
+ copy_self_contained_objects(builder, &compiler, target);
+
+ builder.ensure(StdLink::from_std(self, compiler_to_use));
+ return;
+ }
+
+ target_deps.extend(copy_third_party_objects(builder, &compiler, target));
+ target_deps.extend(copy_self_contained_objects(builder, &compiler, target));
+
+ let mut cargo = builder.cargo(compiler, Mode::Std, SourceType::InTree, target, "build");
+ std_cargo(builder, target, compiler.stage, &mut cargo);
+
+ builder.info(&format!(
+ "Building stage{} std artifacts ({} -> {})",
+ compiler.stage, &compiler.host, target
+ ));
+ run_cargo(
+ builder,
+ cargo,
+ self.crates.to_vec(),
+ &libstd_stamp(builder, compiler, target),
+ target_deps,
+ false,
+ );
+
+ builder.ensure(StdLink::from_std(
+ self,
+ builder.compiler(compiler.stage, builder.config.build),
+ ));
+ }
+}
+
+fn copy_and_stamp(
+ builder: &Builder<'_>,
+ libdir: &Path,
+ sourcedir: &Path,
+ name: &str,
+ target_deps: &mut Vec<(PathBuf, DependencyType)>,
+ dependency_type: DependencyType,
+) {
+ let target = libdir.join(name);
+ builder.copy(&sourcedir.join(name), &target);
+
+ target_deps.push((target, dependency_type));
+}
+
+fn copy_llvm_libunwind(builder: &Builder<'_>, target: TargetSelection, libdir: &Path) -> PathBuf {
+ let libunwind_path = builder.ensure(native::Libunwind { target });
+ let libunwind_source = libunwind_path.join("libunwind.a");
+ let libunwind_target = libdir.join("libunwind.a");
+ builder.copy(&libunwind_source, &libunwind_target);
+ libunwind_target
+}
+
+/// Copies third party objects needed by various targets.
+fn copy_third_party_objects(
+ builder: &Builder<'_>,
+ compiler: &Compiler,
+ target: TargetSelection,
+) -> Vec<(PathBuf, DependencyType)> {
+ let mut target_deps = vec![];
+
+ // FIXME: remove this in 2021
+ if target == "x86_64-fortanix-unknown-sgx" {
+ if env::var_os("X86_FORTANIX_SGX_LIBS").is_some() {
+ builder.info("Warning: X86_FORTANIX_SGX_LIBS environment variable is ignored, libunwind is now compiled as part of rustbuild");
+ }
+ }
+
+ if builder.config.sanitizers_enabled(target) && compiler.stage != 0 {
+ // The sanitizers are only copied in stage1 or above,
+ // to avoid creating dependency on LLVM.
+ target_deps.extend(
+ copy_sanitizers(builder, &compiler, target)
+ .into_iter()
+ .map(|d| (d, DependencyType::Target)),
+ );
+ }
+
+ if target == "x86_64-fortanix-unknown-sgx"
+ || target.contains("pc-windows-gnullvm")
+ || builder.config.llvm_libunwind(target) == LlvmLibunwind::InTree
+ && (target.contains("linux") || target.contains("fuchsia"))
+ {
+ let libunwind_path =
+ copy_llvm_libunwind(builder, target, &builder.sysroot_libdir(*compiler, target));
+ target_deps.push((libunwind_path, DependencyType::Target));
+ }
+
+ target_deps
+}
+
+/// Copies third party objects needed by various targets for self-contained linkage.
+fn copy_self_contained_objects(
+ builder: &Builder<'_>,
+ compiler: &Compiler,
+ target: TargetSelection,
+) -> Vec<(PathBuf, DependencyType)> {
+ let libdir_self_contained = builder.sysroot_libdir(*compiler, target).join("self-contained");
+ t!(fs::create_dir_all(&libdir_self_contained));
+ let mut target_deps = vec![];
+
+ // Copies the libc and CRT objects.
+ //
+ // rustc historically provides a more self-contained installation for musl targets
+ // not requiring the presence of a native musl toolchain. For example, it can fall back
+ // to using gcc from a glibc-targeting toolchain for linking.
+ // To do that we have to distribute musl startup objects as a part of Rust toolchain
+ // and link with them manually in the self-contained mode.
+ if target.contains("musl") {
+ let srcdir = builder.musl_libdir(target).unwrap_or_else(|| {
+ panic!("Target {:?} does not have a \"musl-libdir\" key", target.triple)
+ });
+ for &obj in &["libc.a", "crt1.o", "Scrt1.o", "rcrt1.o", "crti.o", "crtn.o"] {
+ copy_and_stamp(
+ builder,
+ &libdir_self_contained,
+ &srcdir,
+ obj,
+ &mut target_deps,
+ DependencyType::TargetSelfContained,
+ );
+ }
+ let crt_path = builder.ensure(native::CrtBeginEnd { target });
+ for &obj in &["crtbegin.o", "crtbeginS.o", "crtend.o", "crtendS.o"] {
+ let src = crt_path.join(obj);
+ let target = libdir_self_contained.join(obj);
+ builder.copy(&src, &target);
+ target_deps.push((target, DependencyType::TargetSelfContained));
+ }
+
+ if !target.starts_with("s390x") {
+ let libunwind_path = copy_llvm_libunwind(builder, target, &libdir_self_contained);
+ target_deps.push((libunwind_path, DependencyType::TargetSelfContained));
+ }
+ } else if target.ends_with("-wasi") {
+ let srcdir = builder
+ .wasi_root(target)
+ .unwrap_or_else(|| {
+ panic!("Target {:?} does not have a \"wasi-root\" key", target.triple)
+ })
+ .join("lib/wasm32-wasi");
+ for &obj in &["libc.a", "crt1-command.o", "crt1-reactor.o"] {
+ copy_and_stamp(
+ builder,
+ &libdir_self_contained,
+ &srcdir,
+ obj,
+ &mut target_deps,
+ DependencyType::TargetSelfContained,
+ );
+ }
+ } else if target.ends_with("windows-gnu") {
+ for obj in ["crt2.o", "dllcrt2.o"].iter() {
+ let src = compiler_file(builder, builder.cc(target), target, CLang::C, obj);
+ let target = libdir_self_contained.join(obj);
+ builder.copy(&src, &target);
+ target_deps.push((target, DependencyType::TargetSelfContained));
+ }
+ }
+
+ target_deps
+}
+
+/// Configure cargo to compile the standard library, adding appropriate env vars
+/// and such.
+pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, cargo: &mut Cargo) {
+ if let Some(target) = env::var_os("MACOSX_STD_DEPLOYMENT_TARGET") {
+ cargo.env("MACOSX_DEPLOYMENT_TARGET", target);
+ }
+
+ // Determine if we're going to compile in optimized C intrinsics to
+ // the `compiler-builtins` crate. These intrinsics live in LLVM's
+ // `compiler-rt` repository, but our `src/llvm-project` submodule isn't
+ // always checked out, so we need to conditionally look for this. (e.g. if
+ // an external LLVM is used we skip the LLVM submodule checkout).
+ //
+ // Note that this shouldn't affect the correctness of `compiler-builtins`,
+ // but only its speed. Some intrinsics in C haven't been translated to Rust
+ // yet but that's pretty rare. Other intrinsics have optimized
+ // implementations in C which have only had slower versions ported to Rust,
+ // so we favor the C version where we can, but it's not critical.
+ //
+ // If `compiler-rt` is available ensure that the `c` feature of the
+ // `compiler-builtins` crate is enabled and it's configured to learn where
+ // `compiler-rt` is located.
+ let compiler_builtins_root = builder.src.join("src/llvm-project/compiler-rt");
+ let compiler_builtins_c_feature = if compiler_builtins_root.exists() {
+ // Note that `libprofiler_builtins/build.rs` also computes this so if
+ // you're changing something here please also change that.
+ cargo.env("RUST_COMPILER_RT_ROOT", &compiler_builtins_root);
+ " compiler-builtins-c"
+ } else {
+ ""
+ };
+
+ if builder.no_std(target) == Some(true) {
+ let mut features = "compiler-builtins-mem".to_string();
+ if !target.starts_with("bpf") {
+ features.push_str(compiler_builtins_c_feature);
+ }
+
+ // for no-std targets we only compile a few no_std crates
+ cargo
+ .args(&["-p", "alloc"])
+ .arg("--manifest-path")
+ .arg(builder.src.join("library/alloc/Cargo.toml"))
+ .arg("--features")
+ .arg(features);
+ } else {
+ let mut features = builder.std_features(target);
+ features.push_str(compiler_builtins_c_feature);
+
+ cargo
+ .arg("--features")
+ .arg(features)
+ .arg("--manifest-path")
+ .arg(builder.src.join("library/test/Cargo.toml"));
+
+ // Help the libc crate compile by assisting it in finding various
+ // sysroot native libraries.
+ if target.contains("musl") {
+ if let Some(p) = builder.musl_libdir(target) {
+ let root = format!("native={}", p.to_str().unwrap());
+ cargo.rustflag("-L").rustflag(&root);
+ }
+ }
+
+ if target.ends_with("-wasi") {
+ if let Some(p) = builder.wasi_root(target) {
+ let root = format!("native={}/lib/wasm32-wasi", p.to_str().unwrap());
+ cargo.rustflag("-L").rustflag(&root);
+ }
+ }
+ }
+
+ // By default, rustc uses `-Cembed-bitcode=yes`, and Cargo overrides that
+ // with `-Cembed-bitcode=no` for non-LTO builds. However, libstd must be
+ // built with bitcode so that the produced rlibs can be used for both LTO
+ // builds (which use bitcode) and non-LTO builds (which use object code).
+ // So we override the override here!
+ //
+ // But we don't bother for the stage 0 compiler because it's never used
+ // with LTO.
+ if stage >= 1 {
+ cargo.rustflag("-Cembed-bitcode=yes");
+ }
+
+ // By default, rustc does not include unwind tables unless they are required
+ // for a particular target. They are not required by RISC-V targets, but
+ // compiling the standard library with them means that users can get
+ // backtraces without having to recompile the standard library themselves.
+ //
+ // This choice was discussed in https://github.com/rust-lang/rust/pull/69890
+ if target.contains("riscv") {
+ cargo.rustflag("-Cforce-unwind-tables=yes");
+ }
+
+ let html_root =
+ format!("-Zcrate-attr=doc(html_root_url=\"{}/\")", builder.doc_rust_lang_org_channel(),);
+ cargo.rustflag(&html_root);
+ cargo.rustdocflag(&html_root);
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+struct StdLink {
+ pub compiler: Compiler,
+ pub target_compiler: Compiler,
+ pub target: TargetSelection,
+ /// Not actually used; only present to make sure the cache invalidation is correct.
+ crates: Interned<Vec<String>>,
+}
+
+impl StdLink {
+ fn from_std(std: Std, host_compiler: Compiler) -> Self {
+ Self {
+ compiler: host_compiler,
+ target_compiler: std.compiler,
+ target: std.target,
+ crates: std.crates,
+ }
+ }
+}
+
+impl Step for StdLink {
+ type Output = ();
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.never()
+ }
+
+ /// Link all libstd rlibs/dylibs into the sysroot location.
+ ///
+ /// Links those artifacts generated by `compiler` to the `stage` compiler's
+ /// sysroot for the specified `host` and `target`.
+ ///
+ /// Note that this assumes that `compiler` has already generated the libstd
+ /// libraries for `target`, and this method will find them in the relevant
+ /// output directory.
+ fn run(self, builder: &Builder<'_>) {
+ let compiler = self.compiler;
+ let target_compiler = self.target_compiler;
+ let target = self.target;
+ builder.info(&format!(
+ "Copying stage{} std from stage{} ({} -> {} / {})",
+ target_compiler.stage, compiler.stage, &compiler.host, target_compiler.host, target
+ ));
+ let libdir = builder.sysroot_libdir(target_compiler, target);
+ let hostdir = builder.sysroot_libdir(target_compiler, compiler.host);
+ add_to_sysroot(builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target));
+ }
+}
+
+/// Copies sanitizer runtime libraries into target libdir.
+fn copy_sanitizers(
+ builder: &Builder<'_>,
+ compiler: &Compiler,
+ target: TargetSelection,
+) -> Vec<PathBuf> {
+ let runtimes: Vec<native::SanitizerRuntime> = builder.ensure(native::Sanitizers { target });
+
+ if builder.config.dry_run {
+ return Vec::new();
+ }
+
+ let mut target_deps = Vec::new();
+ let libdir = builder.sysroot_libdir(*compiler, target);
+
+ for runtime in &runtimes {
+ let dst = libdir.join(&runtime.name);
+ builder.copy(&runtime.path, &dst);
+
+ if target == "x86_64-apple-darwin" || target == "aarch64-apple-darwin" {
+ // Update the library’s install name to reflect that it has has been renamed.
+ apple_darwin_update_library_name(&dst, &format!("@rpath/{}", &runtime.name));
+ // Upon renaming the install name, the code signature of the file will invalidate,
+ // so we will sign it again.
+ apple_darwin_sign_file(&dst);
+ }
+
+ target_deps.push(dst);
+ }
+
+ target_deps
+}
+
+fn apple_darwin_update_library_name(library_path: &Path, new_name: &str) {
+ let status = Command::new("install_name_tool")
+ .arg("-id")
+ .arg(new_name)
+ .arg(library_path)
+ .status()
+ .expect("failed to execute `install_name_tool`");
+ assert!(status.success());
+}
+
+fn apple_darwin_sign_file(file_path: &Path) {
+ let status = Command::new("codesign")
+ .arg("-f") // Force to rewrite the existing signature
+ .arg("-s")
+ .arg("-")
+ .arg(file_path)
+ .status()
+ .expect("failed to execute `codesign`");
+ assert!(status.success());
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct StartupObjects {
+ pub compiler: Compiler,
+ pub target: TargetSelection,
+}
+
+impl Step for StartupObjects {
+ type Output = Vec<(PathBuf, DependencyType)>;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("library/rtstartup")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(StartupObjects {
+ compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()),
+ target: run.target,
+ });
+ }
+
+ /// Builds and prepare startup objects like rsbegin.o and rsend.o
+ ///
+ /// These are primarily used on Windows right now for linking executables/dlls.
+ /// They don't require any library support as they're just plain old object
+ /// files, so we just use the nightly snapshot compiler to always build them (as
+ /// no other compilers are guaranteed to be available).
+ fn run(self, builder: &Builder<'_>) -> Vec<(PathBuf, DependencyType)> {
+ let for_compiler = self.compiler;
+ let target = self.target;
+ if !target.ends_with("windows-gnu") {
+ return vec![];
+ }
+
+ let mut target_deps = vec![];
+
+ let src_dir = &builder.src.join("library").join("rtstartup");
+ let dst_dir = &builder.native_dir(target).join("rtstartup");
+ let sysroot_dir = &builder.sysroot_libdir(for_compiler, target);
+ t!(fs::create_dir_all(dst_dir));
+
+ for file in &["rsbegin", "rsend"] {
+ let src_file = &src_dir.join(file.to_string() + ".rs");
+ let dst_file = &dst_dir.join(file.to_string() + ".o");
+ if !up_to_date(src_file, dst_file) {
+ let mut cmd = Command::new(&builder.initial_rustc);
+ cmd.env("RUSTC_BOOTSTRAP", "1");
+ if !builder.local_rebuild {
+ // a local_rebuild compiler already has stage1 features
+ cmd.arg("--cfg").arg("bootstrap");
+ }
+ builder.run(
+ cmd.arg("--target")
+ .arg(target.rustc_target_arg())
+ .arg("--emit=obj")
+ .arg("-o")
+ .arg(dst_file)
+ .arg(src_file),
+ );
+ }
+
+ let target = sysroot_dir.join((*file).to_string() + ".o");
+ builder.copy(dst_file, &target);
+ target_deps.push((target, DependencyType::Target));
+ }
+
+ target_deps
+ }
+}
+
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct Rustc {
+ pub target: TargetSelection,
+ pub compiler: Compiler,
+ /// Whether to build a subset of crates, rather than the whole compiler.
+ ///
+ /// This should only be requested by the user, not used within rustbuild itself.
+ /// Using it within rustbuild can lead to confusing situation where lints are replayed
+ /// in two different steps.
+ crates: Interned<Vec<String>>,
+}
+
+impl Rustc {
+ pub fn new(compiler: Compiler, target: TargetSelection) -> Self {
+ Self { target, compiler, crates: Default::default() }
+ }
+}
+
+impl Step for Rustc {
+ type Output = ();
+ const ONLY_HOSTS: bool = true;
+ const DEFAULT: bool = false;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let mut crates = run.builder.in_tree_crates("rustc-main", None);
+ for (i, krate) in crates.iter().enumerate() {
+ if krate.name == "rustc-main" {
+ crates.swap_remove(i);
+ break;
+ }
+ }
+ run.crates(crates)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ let crates = build_crates_in_set(&run);
+ run.builder.ensure(Rustc {
+ compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()),
+ target: run.target,
+ crates,
+ });
+ }
+
+ /// Builds the compiler.
+ ///
+ /// This will build the compiler for a particular stage of the build using
+ /// the `compiler` targeting the `target` architecture. The artifacts
+ /// created will also be linked into the sysroot directory.
+ fn run(self, builder: &Builder<'_>) {
+ let compiler = self.compiler;
+ let target = self.target;
+
+ // NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler,
+ // so its artifacts can't be reused.
+ if builder.download_rustc() && compiler.stage != 0 {
+ // Copy the existing artifacts instead of rebuilding them.
+ // NOTE: this path is only taken for tools linking to rustc-dev.
+ builder.ensure(Sysroot { compiler });
+ return;
+ }
+
+ builder.ensure(Std::new(compiler, target));
+
+ if builder.config.keep_stage.contains(&compiler.stage) {
+ builder.info("Warning: Using a potentially old librustc. This may not behave well.");
+ builder.info("Warning: Use `--keep-stage-std` if you want to rebuild the compiler when it changes");
+ builder.ensure(RustcLink::from_rustc(self, compiler));
+ return;
+ }
+
+ let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
+ if compiler_to_use != compiler {
+ builder.ensure(Rustc::new(compiler_to_use, target));
+ builder
+ .info(&format!("Uplifting stage1 rustc ({} -> {})", builder.config.build, target));
+ builder.ensure(RustcLink::from_rustc(self, compiler_to_use));
+ return;
+ }
+
+ // Ensure that build scripts and proc macros have a std / libproc_macro to link against.
+ builder.ensure(Std::new(
+ builder.compiler(self.compiler.stage, builder.config.build),
+ builder.config.build,
+ ));
+
+ let mut cargo = builder.cargo(compiler, Mode::Rustc, SourceType::InTree, target, "build");
+ rustc_cargo(builder, &mut cargo, target);
+
+ if builder.config.rust_profile_use.is_some()
+ && builder.config.rust_profile_generate.is_some()
+ {
+ panic!("Cannot use and generate PGO profiles at the same time");
+ }
+
+ // With LLD, we can use ICF (identical code folding) to reduce the executable size
+ // of librustc_driver/rustc and to improve i-cache utilization.
+ if builder.config.use_lld {
+ cargo.rustflag("-Clink-args=-Wl,--icf=all");
+ }
+
+ let is_collecting = if let Some(path) = &builder.config.rust_profile_generate {
+ if compiler.stage == 1 {
+ cargo.rustflag(&format!("-Cprofile-generate={}", path));
+ // Apparently necessary to avoid overflowing the counters during
+ // a Cargo build profile
+ cargo.rustflag("-Cllvm-args=-vp-counters-per-site=4");
+ true
+ } else {
+ false
+ }
+ } else if let Some(path) = &builder.config.rust_profile_use {
+ if compiler.stage == 1 {
+ cargo.rustflag(&format!("-Cprofile-use={}", path));
+ cargo.rustflag("-Cllvm-args=-pgo-warn-missing-function");
+ true
+ } else {
+ false
+ }
+ } else {
+ false
+ };
+ if is_collecting {
+ // Ensure paths to Rust sources are relative, not absolute.
+ cargo.rustflag(&format!(
+ "-Cllvm-args=-static-func-strip-dirname-prefix={}",
+ builder.config.src.components().count()
+ ));
+ }
+
+ builder.info(&format!(
+ "Building stage{} compiler artifacts ({} -> {})",
+ compiler.stage, &compiler.host, target
+ ));
+ run_cargo(
+ builder,
+ cargo,
+ self.crates.to_vec(),
+ &librustc_stamp(builder, compiler, target),
+ vec![],
+ false,
+ );
+
+ builder.ensure(RustcLink::from_rustc(
+ self,
+ builder.compiler(compiler.stage, builder.config.build),
+ ));
+ }
+}
+
+pub fn rustc_cargo(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelection) {
+ cargo
+ .arg("--features")
+ .arg(builder.rustc_features(builder.kind))
+ .arg("--manifest-path")
+ .arg(builder.src.join("compiler/rustc/Cargo.toml"));
+ rustc_cargo_env(builder, cargo, target);
+}
+
+pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelection) {
+ // Set some configuration variables picked up by build scripts and
+ // the compiler alike
+ cargo
+ .env("CFG_RELEASE", builder.rust_release())
+ .env("CFG_RELEASE_CHANNEL", &builder.config.channel)
+ .env("CFG_VERSION", builder.rust_version());
+
+ if let Some(backend) = builder.config.rust_codegen_backends.get(0) {
+ cargo.env("CFG_DEFAULT_CODEGEN_BACKEND", backend);
+ }
+
+ let libdir_relative = builder.config.libdir_relative().unwrap_or_else(|| Path::new("lib"));
+ let target_config = builder.config.target_config.get(&target);
+
+ cargo.env("CFG_LIBDIR_RELATIVE", libdir_relative);
+
+ if let Some(ref ver_date) = builder.rust_info.commit_date() {
+ cargo.env("CFG_VER_DATE", ver_date);
+ }
+ if let Some(ref ver_hash) = builder.rust_info.sha() {
+ cargo.env("CFG_VER_HASH", ver_hash);
+ }
+ if !builder.unstable_features() {
+ cargo.env("CFG_DISABLE_UNSTABLE_FEATURES", "1");
+ }
+
+ // Prefer the current target's own default_linker, else a globally
+ // specified one.
+ if let Some(s) = target_config.and_then(|c| c.default_linker.as_ref()) {
+ cargo.env("CFG_DEFAULT_LINKER", s);
+ } else if let Some(ref s) = builder.config.rustc_default_linker {
+ cargo.env("CFG_DEFAULT_LINKER", s);
+ }
+
+ if builder.config.rustc_parallel {
+ // keep in sync with `bootstrap/lib.rs:Build::rustc_features`
+ // `cfg` option for rustc, `features` option for cargo, for conditional compilation
+ cargo.rustflag("--cfg=parallel_compiler");
+ cargo.rustdocflag("--cfg=parallel_compiler");
+ }
+ if builder.config.rust_verify_llvm_ir {
+ cargo.env("RUSTC_VERIFY_LLVM_IR", "1");
+ }
+
+ // Pass down configuration from the LLVM build into the build of
+ // rustc_llvm and rustc_codegen_llvm.
+ //
+ // Note that this is disabled if LLVM itself is disabled or we're in a check
+ // build. If we are in a check build we still go ahead here presuming we've
+ // detected that LLVM is already built and good to go which helps prevent
+ // busting caches (e.g. like #71152).
+ if builder.config.llvm_enabled()
+ && (builder.kind != Kind::Check
+ || crate::native::prebuilt_llvm_config(builder, target).is_ok())
+ {
+ if builder.is_rust_llvm(target) {
+ cargo.env("LLVM_RUSTLLVM", "1");
+ }
+ let llvm_config = builder.ensure(native::Llvm { target });
+ cargo.env("LLVM_CONFIG", &llvm_config);
+ if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) {
+ cargo.env("CFG_LLVM_ROOT", s);
+ }
+
+ // Some LLVM linker flags (-L and -l) may be needed to link `rustc_llvm`. Its build script
+ // expects these to be passed via the `LLVM_LINKER_FLAGS` env variable, separated by
+ // whitespace.
+ //
+ // For example:
+ // - on windows, when `clang-cl` is used with instrumentation, we need to manually add
+ // clang's runtime library resource directory so that the profiler runtime library can be
+ // found. This is to avoid the linker errors about undefined references to
+ // `__llvm_profile_instrument_memop` when linking `rustc_driver`.
+ let mut llvm_linker_flags = String::new();
+ if builder.config.llvm_profile_generate && target.contains("msvc") {
+ if let Some(ref clang_cl_path) = builder.config.llvm_clang_cl {
+ // Add clang's runtime library directory to the search path
+ let clang_rt_dir = get_clang_cl_resource_dir(clang_cl_path);
+ llvm_linker_flags.push_str(&format!("-L{}", clang_rt_dir.display()));
+ }
+ }
+
+ // The config can also specify its own llvm linker flags.
+ if let Some(ref s) = builder.config.llvm_ldflags {
+ if !llvm_linker_flags.is_empty() {
+ llvm_linker_flags.push_str(" ");
+ }
+ llvm_linker_flags.push_str(s);
+ }
+
+ // Set the linker flags via the env var that `rustc_llvm`'s build script will read.
+ if !llvm_linker_flags.is_empty() {
+ cargo.env("LLVM_LINKER_FLAGS", llvm_linker_flags);
+ }
+
+ // Building with a static libstdc++ is only supported on linux right now,
+ // not for MSVC or macOS
+ if builder.config.llvm_static_stdcpp
+ && !target.contains("freebsd")
+ && !target.contains("msvc")
+ && !target.contains("apple")
+ && !target.contains("solaris")
+ {
+ let file = compiler_file(
+ builder,
+ builder.cxx(target).unwrap(),
+ target,
+ CLang::Cxx,
+ "libstdc++.a",
+ );
+ cargo.env("LLVM_STATIC_STDCPP", file);
+ }
+ if builder.llvm_link_shared() {
+ cargo.env("LLVM_LINK_SHARED", "1");
+ }
+ if builder.config.llvm_use_libcxx {
+ cargo.env("LLVM_USE_LIBCXX", "1");
+ }
+ if builder.config.llvm_optimize && !builder.config.llvm_release_debuginfo {
+ cargo.env("LLVM_NDEBUG", "1");
+ }
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+struct RustcLink {
+ pub compiler: Compiler,
+ pub target_compiler: Compiler,
+ pub target: TargetSelection,
+ /// Not actually used; only present to make sure the cache invalidation is correct.
+ crates: Interned<Vec<String>>,
+}
+
+impl RustcLink {
+ fn from_rustc(rustc: Rustc, host_compiler: Compiler) -> Self {
+ Self {
+ compiler: host_compiler,
+ target_compiler: rustc.compiler,
+ target: rustc.target,
+ crates: rustc.crates,
+ }
+ }
+}
+
+impl Step for RustcLink {
+ type Output = ();
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.never()
+ }
+
+ /// Same as `std_link`, only for librustc
+ fn run(self, builder: &Builder<'_>) {
+ let compiler = self.compiler;
+ let target_compiler = self.target_compiler;
+ let target = self.target;
+ builder.info(&format!(
+ "Copying stage{} rustc from stage{} ({} -> {} / {})",
+ target_compiler.stage, compiler.stage, &compiler.host, target_compiler.host, target
+ ));
+ add_to_sysroot(
+ builder,
+ &builder.sysroot_libdir(target_compiler, target),
+ &builder.sysroot_libdir(target_compiler, compiler.host),
+ &librustc_stamp(builder, compiler, target),
+ );
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct CodegenBackend {
+ pub target: TargetSelection,
+ pub compiler: Compiler,
+ pub backend: Interned<String>,
+}
+
+impl Step for CodegenBackend {
+ type Output = ();
+ const ONLY_HOSTS: bool = true;
+ // Only the backends specified in the `codegen-backends` entry of `config.toml` are built.
+ const DEFAULT: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.paths(&["compiler/rustc_codegen_cranelift", "compiler/rustc_codegen_gcc"])
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ for &backend in &run.builder.config.rust_codegen_backends {
+ if backend == "llvm" {
+ continue; // Already built as part of rustc
+ }
+
+ run.builder.ensure(CodegenBackend {
+ target: run.target,
+ compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()),
+ backend,
+ });
+ }
+ }
+
+ fn run(self, builder: &Builder<'_>) {
+ let compiler = self.compiler;
+ let target = self.target;
+ let backend = self.backend;
+
+ builder.ensure(Rustc::new(compiler, target));
+
+ if builder.config.keep_stage.contains(&compiler.stage) {
+ builder.info(
+ "Warning: Using a potentially old codegen backend. \
+ This may not behave well.",
+ );
+ // Codegen backends are linked separately from this step today, so we don't do
+ // anything here.
+ return;
+ }
+
+ let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
+ if compiler_to_use != compiler {
+ builder.ensure(CodegenBackend { compiler: compiler_to_use, target, backend });
+ return;
+ }
+
+ let out_dir = builder.cargo_out(compiler, Mode::Codegen, target);
+
+ let mut cargo = builder.cargo(compiler, Mode::Codegen, SourceType::InTree, target, "build");
+ cargo
+ .arg("--manifest-path")
+ .arg(builder.src.join(format!("compiler/rustc_codegen_{}/Cargo.toml", backend)));
+ rustc_cargo_env(builder, &mut cargo, target);
+
+ let tmp_stamp = out_dir.join(".tmp.stamp");
+
+ builder.info(&format!(
+ "Building stage{} codegen backend {} ({} -> {})",
+ compiler.stage, backend, &compiler.host, target
+ ));
+ let files = run_cargo(builder, cargo, vec![], &tmp_stamp, vec![], false);
+ if builder.config.dry_run {
+ return;
+ }
+ let mut files = files.into_iter().filter(|f| {
+ let filename = f.file_name().unwrap().to_str().unwrap();
+ is_dylib(filename) && filename.contains("rustc_codegen_")
+ });
+ let codegen_backend = match files.next() {
+ Some(f) => f,
+ None => panic!("no dylibs built for codegen backend?"),
+ };
+ if let Some(f) = files.next() {
+ panic!(
+ "codegen backend built two dylibs:\n{}\n{}",
+ codegen_backend.display(),
+ f.display()
+ );
+ }
+ let stamp = codegen_backend_stamp(builder, compiler, target, backend);
+ let codegen_backend = codegen_backend.to_str().unwrap();
+ t!(fs::write(&stamp, &codegen_backend));
+ }
+}
+
+/// Creates the `codegen-backends` folder for a compiler that's about to be
+/// assembled as a complete compiler.
+///
+/// This will take the codegen artifacts produced by `compiler` and link them
+/// into an appropriate location for `target_compiler` to be a functional
+/// compiler.
+fn copy_codegen_backends_to_sysroot(
+ builder: &Builder<'_>,
+ compiler: Compiler,
+ target_compiler: Compiler,
+) {
+ let target = target_compiler.host;
+
+ // Note that this step is different than all the other `*Link` steps in
+ // that it's not assembling a bunch of libraries but rather is primarily
+ // moving the codegen backend into place. The codegen backend of rustc is
+ // not linked into the main compiler by default but is rather dynamically
+ // selected at runtime for inclusion.
+ //
+ // Here we're looking for the output dylib of the `CodegenBackend` step and
+ // we're copying that into the `codegen-backends` folder.
+ let dst = builder.sysroot_codegen_backends(target_compiler);
+ t!(fs::create_dir_all(&dst), dst);
+
+ if builder.config.dry_run {
+ return;
+ }
+
+ for backend in builder.config.rust_codegen_backends.iter() {
+ if backend == "llvm" {
+ continue; // Already built as part of rustc
+ }
+
+ let stamp = codegen_backend_stamp(builder, compiler, target, *backend);
+ let dylib = t!(fs::read_to_string(&stamp));
+ let file = Path::new(&dylib);
+ let filename = file.file_name().unwrap().to_str().unwrap();
+ // change `librustc_codegen_cranelift-xxxxxx.so` to
+ // `librustc_codegen_cranelift-release.so`
+ let target_filename = {
+ let dash = filename.find('-').unwrap();
+ let dot = filename.find('.').unwrap();
+ format!("{}-{}{}", &filename[..dash], builder.rust_release(), &filename[dot..])
+ };
+ builder.copy(&file, &dst.join(target_filename));
+ }
+}
+
+/// Cargo's output path for the standard library in a given stage, compiled
+/// by a particular compiler for the specified target.
+pub fn libstd_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf {
+ builder.cargo_out(compiler, Mode::Std, target).join(".libstd.stamp")
+}
+
+/// Cargo's output path for librustc in a given stage, compiled by a particular
+/// compiler for the specified target.
+pub fn librustc_stamp(
+ builder: &Builder<'_>,
+ compiler: Compiler,
+ target: TargetSelection,
+) -> PathBuf {
+ builder.cargo_out(compiler, Mode::Rustc, target).join(".librustc.stamp")
+}
+
+/// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular
+/// compiler for the specified target and backend.
+fn codegen_backend_stamp(
+ builder: &Builder<'_>,
+ compiler: Compiler,
+ target: TargetSelection,
+ backend: Interned<String>,
+) -> PathBuf {
+ builder
+ .cargo_out(compiler, Mode::Codegen, target)
+ .join(format!(".librustc_codegen_{}.stamp", backend))
+}
+
+pub fn compiler_file(
+ builder: &Builder<'_>,
+ compiler: &Path,
+ target: TargetSelection,
+ c: CLang,
+ file: &str,
+) -> PathBuf {
+ let mut cmd = Command::new(compiler);
+ cmd.args(builder.cflags(target, GitRepo::Rustc, c));
+ cmd.arg(format!("-print-file-name={}", file));
+ let out = output(&mut cmd);
+ PathBuf::from(out.trim())
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct Sysroot {
+ pub compiler: Compiler,
+}
+
+impl Step for Sysroot {
+ type Output = Interned<PathBuf>;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.never()
+ }
+
+ /// Returns the sysroot for the `compiler` specified that *this build system
+ /// generates*.
+ ///
+ /// That is, the sysroot for the stage0 compiler is not what the compiler
+ /// thinks it is by default, but it's the same as the default for stages
+ /// 1-3.
+ fn run(self, builder: &Builder<'_>) -> Interned<PathBuf> {
+ let compiler = self.compiler;
+ let sysroot = if compiler.stage == 0 {
+ builder.out.join(&compiler.host.triple).join("stage0-sysroot")
+ } else {
+ builder.out.join(&compiler.host.triple).join(format!("stage{}", compiler.stage))
+ };
+ let _ = fs::remove_dir_all(&sysroot);
+ t!(fs::create_dir_all(&sysroot));
+
+ // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0.
+ if builder.download_rustc() && compiler.stage != 0 {
+ assert_eq!(
+ builder.config.build, compiler.host,
+ "Cross-compiling is not yet supported with `download-rustc`",
+ );
+ // Copy the compiler into the correct sysroot.
+ let ci_rustc_dir =
+ builder.config.out.join(&*builder.config.build.triple).join("ci-rustc");
+ builder.cp_r(&ci_rustc_dir, &sysroot);
+ return INTERNER.intern_path(sysroot);
+ }
+
+ // Symlink the source root into the same location inside the sysroot,
+ // where `rust-src` component would go (`$sysroot/lib/rustlib/src/rust`),
+ // so that any tools relying on `rust-src` also work for local builds,
+ // and also for translating the virtual `/rustc/$hash` back to the real
+ // directory (for running tests with `rust.remap-debuginfo = true`).
+ let sysroot_lib_rustlib_src = sysroot.join("lib/rustlib/src");
+ t!(fs::create_dir_all(&sysroot_lib_rustlib_src));
+ let sysroot_lib_rustlib_src_rust = sysroot_lib_rustlib_src.join("rust");
+ if let Err(e) = symlink_dir(&builder.config, &builder.src, &sysroot_lib_rustlib_src_rust) {
+ eprintln!(
+ "warning: creating symbolic link `{}` to `{}` failed with {}",
+ sysroot_lib_rustlib_src_rust.display(),
+ builder.src.display(),
+ e,
+ );
+ if builder.config.rust_remap_debuginfo {
+ eprintln!(
+ "warning: some `src/test/ui` tests will fail when lacking `{}`",
+ sysroot_lib_rustlib_src_rust.display(),
+ );
+ }
+ }
+
+ INTERNER.intern_path(sysroot)
+ }
+}
+
+#[derive(Debug, Copy, PartialOrd, Ord, Clone, PartialEq, Eq, Hash)]
+pub struct Assemble {
+ /// The compiler which we will produce in this step. Assemble itself will
+ /// take care of ensuring that the necessary prerequisites to do so exist,
+ /// that is, this target can be a stage2 compiler and Assemble will build
+ /// previous stages for you.
+ pub target_compiler: Compiler,
+}
+
+impl Step for Assemble {
+ type Output = Compiler;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("compiler/rustc").path("compiler")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Assemble {
+ target_compiler: run.builder.compiler(run.builder.top_stage + 1, run.target),
+ });
+ }
+
+ /// Prepare a new compiler from the artifacts in `stage`
+ ///
+ /// This will assemble a compiler in `build/$host/stage$stage`. The compiler
+ /// must have been previously produced by the `stage - 1` builder.build
+ /// compiler.
+ fn run(self, builder: &Builder<'_>) -> Compiler {
+ let target_compiler = self.target_compiler;
+
+ if target_compiler.stage == 0 {
+ assert_eq!(
+ builder.config.build, target_compiler.host,
+ "Cannot obtain compiler for non-native build triple at stage 0"
+ );
+ // The stage 0 compiler for the build triple is always pre-built.
+ return target_compiler;
+ }
+
+ // Get the compiler that we'll use to bootstrap ourselves.
+ //
+ // Note that this is where the recursive nature of the bootstrap
+ // happens, as this will request the previous stage's compiler on
+ // downwards to stage 0.
+ //
+ // Also note that we're building a compiler for the host platform. We
+ // only assume that we can run `build` artifacts, which means that to
+ // produce some other architecture compiler we need to start from
+ // `build` to get there.
+ //
+ // FIXME: It may be faster if we build just a stage 1 compiler and then
+ // use that to bootstrap this compiler forward.
+ let build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build);
+
+ // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0.
+ if builder.download_rustc() {
+ builder.ensure(Sysroot { compiler: target_compiler });
+ return target_compiler;
+ }
+
+ // Build the libraries for this compiler to link to (i.e., the libraries
+ // it uses at runtime). NOTE: Crates the target compiler compiles don't
+ // link to these. (FIXME: Is that correct? It seems to be correct most
+ // of the time but I think we do link to these for stage2/bin compilers
+ // when not performing a full bootstrap).
+ builder.ensure(Rustc::new(build_compiler, target_compiler.host));
+
+ for &backend in builder.config.rust_codegen_backends.iter() {
+ if backend == "llvm" {
+ continue; // Already built as part of rustc
+ }
+
+ builder.ensure(CodegenBackend {
+ compiler: build_compiler,
+ target: target_compiler.host,
+ backend,
+ });
+ }
+
+ let lld_install = if builder.config.lld_enabled {
+ Some(builder.ensure(native::Lld { target: target_compiler.host }))
+ } else {
+ None
+ };
+
+ let stage = target_compiler.stage;
+ let host = target_compiler.host;
+ builder.info(&format!("Assembling stage{} compiler ({})", stage, host));
+
+ // Link in all dylibs to the libdir
+ let stamp = librustc_stamp(builder, build_compiler, target_compiler.host);
+ let proc_macros = builder
+ .read_stamp_file(&stamp)
+ .into_iter()
+ .filter_map(|(path, dependency_type)| {
+ if dependency_type == DependencyType::Host {
+ Some(path.file_name().unwrap().to_owned().into_string().unwrap())
+ } else {
+ None
+ }
+ })
+ .collect::<HashSet<_>>();
+
+ let sysroot = builder.sysroot(target_compiler);
+ let rustc_libdir = builder.rustc_libdir(target_compiler);
+ t!(fs::create_dir_all(&rustc_libdir));
+ let src_libdir = builder.sysroot_libdir(build_compiler, host);
+ for f in builder.read_dir(&src_libdir) {
+ let filename = f.file_name().into_string().unwrap();
+ if (is_dylib(&filename) || is_debug_info(&filename)) && !proc_macros.contains(&filename)
+ {
+ builder.copy(&f.path(), &rustc_libdir.join(&filename));
+ }
+ }
+
+ copy_codegen_backends_to_sysroot(builder, build_compiler, target_compiler);
+
+ // We prepend this bin directory to the user PATH when linking Rust binaries. To
+ // avoid shadowing the system LLD we rename the LLD we provide to `rust-lld`.
+ let libdir = builder.sysroot_libdir(target_compiler, target_compiler.host);
+ let libdir_bin = libdir.parent().unwrap().join("bin");
+ t!(fs::create_dir_all(&libdir_bin));
+ if let Some(lld_install) = lld_install {
+ let src_exe = exe("lld", target_compiler.host);
+ let dst_exe = exe("rust-lld", target_compiler.host);
+ builder.copy(&lld_install.join("bin").join(&src_exe), &libdir_bin.join(&dst_exe));
+ // for `-Z gcc-ld=lld`
+ let gcc_ld_dir = libdir_bin.join("gcc-ld");
+ t!(fs::create_dir(&gcc_ld_dir));
+ let lld_wrapper_exe = builder.ensure(crate::tool::LldWrapper {
+ compiler: build_compiler,
+ target: target_compiler.host,
+ });
+ builder.copy(&lld_wrapper_exe, &gcc_ld_dir.join(exe("ld", target_compiler.host)));
+ }
+
+ if builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) {
+ let llvm_config_bin = builder.ensure(native::Llvm { target: target_compiler.host });
+ if !builder.config.dry_run {
+ let llvm_bin_dir = output(Command::new(llvm_config_bin).arg("--bindir"));
+ let llvm_bin_dir = Path::new(llvm_bin_dir.trim());
+
+ // Since we've already built the LLVM tools, install them to the sysroot.
+ // This is the equivalent of installing the `llvm-tools-preview` component via
+ // rustup, and lets developers use a locally built toolchain to
+ // build projects that expect llvm tools to be present in the sysroot
+ // (e.g. the `bootimage` crate).
+ for tool in LLVM_TOOLS {
+ let tool_exe = exe(tool, target_compiler.host);
+ let src_path = llvm_bin_dir.join(&tool_exe);
+ // When using `download-ci-llvm`, some of the tools
+ // may not exist, so skip trying to copy them.
+ if src_path.exists() {
+ builder.copy(&src_path, &libdir_bin.join(&tool_exe));
+ }
+ }
+ }
+ }
+
+ // Ensure that `libLLVM.so` ends up in the newly build compiler directory,
+ // so that it can be found when the newly built `rustc` is run.
+ dist::maybe_install_llvm_runtime(builder, target_compiler.host, &sysroot);
+ dist::maybe_install_llvm_target(builder, target_compiler.host, &sysroot);
+
+ // Link the compiler binary itself into place
+ let out_dir = builder.cargo_out(build_compiler, Mode::Rustc, host);
+ let rustc = out_dir.join(exe("rustc-main", host));
+ let bindir = sysroot.join("bin");
+ t!(fs::create_dir_all(&bindir));
+ let compiler = builder.rustc(target_compiler);
+ builder.copy(&rustc, &compiler);
+
+ target_compiler
+ }
+}
+
+/// Link some files into a rustc sysroot.
+///
+/// For a particular stage this will link the file listed in `stamp` into the
+/// `sysroot_dst` provided.
+pub fn add_to_sysroot(
+ builder: &Builder<'_>,
+ sysroot_dst: &Path,
+ sysroot_host_dst: &Path,
+ stamp: &Path,
+) {
+ let self_contained_dst = &sysroot_dst.join("self-contained");
+ t!(fs::create_dir_all(&sysroot_dst));
+ t!(fs::create_dir_all(&sysroot_host_dst));
+ t!(fs::create_dir_all(&self_contained_dst));
+ for (path, dependency_type) in builder.read_stamp_file(stamp) {
+ let dst = match dependency_type {
+ DependencyType::Host => sysroot_host_dst,
+ DependencyType::Target => sysroot_dst,
+ DependencyType::TargetSelfContained => self_contained_dst,
+ };
+ builder.copy(&path, &dst.join(path.file_name().unwrap()));
+ }
+}
+
+pub fn run_cargo(
+ builder: &Builder<'_>,
+ cargo: Cargo,
+ tail_args: Vec<String>,
+ stamp: &Path,
+ additional_target_deps: Vec<(PathBuf, DependencyType)>,
+ is_check: bool,
+) -> Vec<PathBuf> {
+ if builder.config.dry_run {
+ return Vec::new();
+ }
+
+ // `target_root_dir` looks like $dir/$target/release
+ let target_root_dir = stamp.parent().unwrap();
+ // `target_deps_dir` looks like $dir/$target/release/deps
+ let target_deps_dir = target_root_dir.join("deps");
+ // `host_root_dir` looks like $dir/release
+ let host_root_dir = target_root_dir
+ .parent()
+ .unwrap() // chop off `release`
+ .parent()
+ .unwrap() // chop off `$target`
+ .join(target_root_dir.file_name().unwrap());
+
+ // Spawn Cargo slurping up its JSON output. We'll start building up the
+ // `deps` array of all files it generated along with a `toplevel` array of
+ // files we need to probe for later.
+ let mut deps = Vec::new();
+ let mut toplevel = Vec::new();
+ let ok = stream_cargo(builder, cargo, tail_args, &mut |msg| {
+ let (filenames, crate_types) = match msg {
+ CargoMessage::CompilerArtifact {
+ filenames,
+ target: CargoTarget { crate_types },
+ ..
+ } => (filenames, crate_types),
+ _ => return,
+ };
+ for filename in filenames {
+ // Skip files like executables
+ if !(filename.ends_with(".rlib")
+ || filename.ends_with(".lib")
+ || filename.ends_with(".a")
+ || is_debug_info(&filename)
+ || is_dylib(&filename)
+ || (is_check && filename.ends_with(".rmeta")))
+ {
+ continue;
+ }
+
+ let filename = Path::new(&*filename);
+
+ // If this was an output file in the "host dir" we don't actually
+ // worry about it, it's not relevant for us
+ if filename.starts_with(&host_root_dir) {
+ // Unless it's a proc macro used in the compiler
+ if crate_types.iter().any(|t| t == "proc-macro") {
+ deps.push((filename.to_path_buf(), DependencyType::Host));
+ }
+ continue;
+ }
+
+ // If this was output in the `deps` dir then this is a precise file
+ // name (hash included) so we start tracking it.
+ if filename.starts_with(&target_deps_dir) {
+ deps.push((filename.to_path_buf(), DependencyType::Target));
+ continue;
+ }
+
+ // Otherwise this was a "top level artifact" which right now doesn't
+ // have a hash in the name, but there's a version of this file in
+ // the `deps` folder which *does* have a hash in the name. That's
+ // the one we'll want to we'll probe for it later.
+ //
+ // We do not use `Path::file_stem` or `Path::extension` here,
+ // because some generated files may have multiple extensions e.g.
+ // `std-<hash>.dll.lib` on Windows. The aforementioned methods only
+ // split the file name by the last extension (`.lib`) while we need
+ // to split by all extensions (`.dll.lib`).
+ let expected_len = t!(filename.metadata()).len();
+ let filename = filename.file_name().unwrap().to_str().unwrap();
+ let mut parts = filename.splitn(2, '.');
+ let file_stem = parts.next().unwrap().to_owned();
+ let extension = parts.next().unwrap().to_owned();
+
+ toplevel.push((file_stem, extension, expected_len));
+ }
+ });
+
+ if !ok {
+ crate::detail_exit(1);
+ }
+
+ // Ok now we need to actually find all the files listed in `toplevel`. We've
+ // got a list of prefix/extensions and we basically just need to find the
+ // most recent file in the `deps` folder corresponding to each one.
+ let contents = t!(target_deps_dir.read_dir())
+ .map(|e| t!(e))
+ .map(|e| (e.path(), e.file_name().into_string().unwrap(), t!(e.metadata())))
+ .collect::<Vec<_>>();
+ for (prefix, extension, expected_len) in toplevel {
+ let candidates = contents.iter().filter(|&&(_, ref filename, ref meta)| {
+ meta.len() == expected_len
+ && filename
+ .strip_prefix(&prefix[..])
+ .map(|s| s.starts_with('-') && s.ends_with(&extension[..]))
+ .unwrap_or(false)
+ });
+ let max = candidates.max_by_key(|&&(_, _, ref metadata)| {
+ metadata.modified().expect("mtime should be available on all relevant OSes")
+ });
+ let path_to_add = match max {
+ Some(triple) => triple.0.to_str().unwrap(),
+ None => panic!("no output generated for {:?} {:?}", prefix, extension),
+ };
+ if is_dylib(path_to_add) {
+ let candidate = format!("{}.lib", path_to_add);
+ let candidate = PathBuf::from(candidate);
+ if candidate.exists() {
+ deps.push((candidate, DependencyType::Target));
+ }
+ }
+ deps.push((path_to_add.into(), DependencyType::Target));
+ }
+
+ deps.extend(additional_target_deps);
+ deps.sort();
+ let mut new_contents = Vec::new();
+ for (dep, dependency_type) in deps.iter() {
+ new_contents.extend(match *dependency_type {
+ DependencyType::Host => b"h",
+ DependencyType::Target => b"t",
+ DependencyType::TargetSelfContained => b"s",
+ });
+ new_contents.extend(dep.to_str().unwrap().as_bytes());
+ new_contents.extend(b"\0");
+ }
+ t!(fs::write(&stamp, &new_contents));
+ deps.into_iter().map(|(d, _)| d).collect()
+}
+
+pub fn stream_cargo(
+ builder: &Builder<'_>,
+ cargo: Cargo,
+ tail_args: Vec<String>,
+ cb: &mut dyn FnMut(CargoMessage<'_>),
+) -> bool {
+ let mut cargo = Command::from(cargo);
+ if builder.config.dry_run {
+ return true;
+ }
+ // Instruct Cargo to give us json messages on stdout, critically leaving
+ // stderr as piped so we can get those pretty colors.
+ let mut message_format = if builder.config.json_output {
+ String::from("json")
+ } else {
+ String::from("json-render-diagnostics")
+ };
+ if let Some(s) = &builder.config.rustc_error_format {
+ message_format.push_str(",json-diagnostic-");
+ message_format.push_str(s);
+ }
+ cargo.arg("--message-format").arg(message_format).stdout(Stdio::piped());
+
+ for arg in tail_args {
+ cargo.arg(arg);
+ }
+
+ builder.verbose(&format!("running: {:?}", cargo));
+ let mut child = match cargo.spawn() {
+ Ok(child) => child,
+ Err(e) => panic!("failed to execute command: {:?}\nerror: {}", cargo, e),
+ };
+
+ // Spawn Cargo slurping up its JSON output. We'll start building up the
+ // `deps` array of all files it generated along with a `toplevel` array of
+ // files we need to probe for later.
+ let stdout = BufReader::new(child.stdout.take().unwrap());
+ for line in stdout.lines() {
+ let line = t!(line);
+ match serde_json::from_str::<CargoMessage<'_>>(&line) {
+ Ok(msg) => {
+ if builder.config.json_output {
+ // Forward JSON to stdout.
+ println!("{}", line);
+ }
+ cb(msg)
+ }
+ // If this was informational, just print it out and continue
+ Err(_) => println!("{}", line),
+ }
+ }
+
+ // Make sure Cargo actually succeeded after we read all of its stdout.
+ let status = t!(child.wait());
+ if builder.is_verbose() && !status.success() {
+ eprintln!(
+ "command did not execute successfully: {:?}\n\
+ expected success, got: {}",
+ cargo, status
+ );
+ }
+ status.success()
+}
+
+#[derive(Deserialize)]
+pub struct CargoTarget<'a> {
+ crate_types: Vec<Cow<'a, str>>,
+}
+
+#[derive(Deserialize)]
+#[serde(tag = "reason", rename_all = "kebab-case")]
+pub enum CargoMessage<'a> {
+ CompilerArtifact {
+ package_id: Cow<'a, str>,
+ features: Vec<Cow<'a, str>>,
+ filenames: Vec<Cow<'a, str>>,
+ target: CargoTarget<'a>,
+ },
+ BuildScriptExecuted {
+ package_id: Cow<'a, str>,
+ },
+ BuildFinished {
+ success: bool,
+ },
+}
diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs
new file mode 100644
index 000000000..4325a237c
--- /dev/null
+++ b/src/bootstrap/config.rs
@@ -0,0 +1,1668 @@
+//! Serialized configuration of a build.
+//!
+//! This module implements parsing `config.toml` configuration files to tweak
+//! how the build runs.
+
+use std::cell::{Cell, RefCell};
+use std::cmp;
+use std::collections::{HashMap, HashSet};
+use std::env;
+use std::ffi::OsStr;
+use std::fmt;
+use std::fs;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+use std::str::FromStr;
+
+use crate::builder::{Builder, TaskPath};
+use crate::cache::{Interned, INTERNER};
+use crate::channel::GitInfo;
+pub use crate::flags::Subcommand;
+use crate::flags::{Color, Flags};
+use crate::util::{exe, output, program_out_of_date, t};
+use once_cell::sync::OnceCell;
+use serde::{Deserialize, Deserializer};
+
+macro_rules! check_ci_llvm {
+ ($name:expr) => {
+ assert!(
+ $name.is_none(),
+ "setting {} is incompatible with download-ci-llvm.",
+ stringify!($name)
+ );
+ };
+}
+
+/// Global configuration for the entire build and/or bootstrap.
+///
+/// This structure is derived from a combination of both `config.toml` and
+/// `config.mk`. As of the time of this writing it's unlikely that `config.toml`
+/// is used all that much, so this is primarily filled out by `config.mk` which
+/// is generated from `./configure`.
+///
+/// Note that this structure is not decoded directly into, but rather it is
+/// filled out from the decoded forms of the structs below. For documentation
+/// each field, see the corresponding fields in
+/// `config.toml.example`.
+#[derive(Default)]
+#[cfg_attr(test, derive(Clone))]
+pub struct Config {
+ pub changelog_seen: Option<usize>,
+ pub ccache: Option<String>,
+ /// Call Build::ninja() instead of this.
+ pub ninja_in_file: bool,
+ pub verbose: usize,
+ pub submodules: Option<bool>,
+ pub compiler_docs: bool,
+ pub docs_minification: bool,
+ pub docs: bool,
+ pub locked_deps: bool,
+ pub vendor: bool,
+ pub target_config: HashMap<TargetSelection, Target>,
+ pub full_bootstrap: bool,
+ pub extended: bool,
+ pub tools: Option<HashSet<String>>,
+ pub sanitizers: bool,
+ pub profiler: bool,
+ pub ignore_git: bool,
+ pub exclude: Vec<TaskPath>,
+ pub include_default_paths: bool,
+ pub rustc_error_format: Option<String>,
+ pub json_output: bool,
+ pub test_compare_mode: bool,
+ pub color: Color,
+ pub patch_binaries_for_nix: bool,
+ pub stage0_metadata: Stage0Metadata,
+
+ pub on_fail: Option<String>,
+ pub stage: u32,
+ pub keep_stage: Vec<u32>,
+ pub keep_stage_std: Vec<u32>,
+ pub src: PathBuf,
+ /// defaults to `config.toml`
+ pub config: PathBuf,
+ pub jobs: Option<u32>,
+ pub cmd: Subcommand,
+ pub incremental: bool,
+ pub dry_run: bool,
+ /// `None` if we shouldn't download CI compiler artifacts, or the commit to download if we should.
+ #[cfg(not(test))]
+ download_rustc_commit: Option<String>,
+ #[cfg(test)]
+ pub download_rustc_commit: Option<String>,
+
+ pub deny_warnings: bool,
+ pub backtrace_on_ice: bool,
+
+ // llvm codegen options
+ pub llvm_skip_rebuild: bool,
+ pub llvm_assertions: bool,
+ pub llvm_tests: bool,
+ pub llvm_plugins: bool,
+ pub llvm_optimize: bool,
+ pub llvm_thin_lto: bool,
+ pub llvm_release_debuginfo: bool,
+ pub llvm_version_check: bool,
+ pub llvm_static_stdcpp: bool,
+ /// `None` if `llvm_from_ci` is true and we haven't yet downloaded llvm.
+ #[cfg(not(test))]
+ llvm_link_shared: Cell<Option<bool>>,
+ #[cfg(test)]
+ pub llvm_link_shared: Cell<Option<bool>>,
+ pub llvm_clang_cl: Option<String>,
+ pub llvm_targets: Option<String>,
+ pub llvm_experimental_targets: Option<String>,
+ pub llvm_link_jobs: Option<u32>,
+ pub llvm_version_suffix: Option<String>,
+ pub llvm_use_linker: Option<String>,
+ pub llvm_allow_old_toolchain: bool,
+ pub llvm_polly: bool,
+ pub llvm_clang: bool,
+ pub llvm_from_ci: bool,
+ pub llvm_build_config: HashMap<String, String>,
+
+ pub use_lld: bool,
+ pub lld_enabled: bool,
+ pub llvm_tools_enabled: bool,
+
+ pub llvm_cflags: Option<String>,
+ pub llvm_cxxflags: Option<String>,
+ pub llvm_ldflags: Option<String>,
+ pub llvm_use_libcxx: bool,
+
+ // rust codegen options
+ pub rust_optimize: bool,
+ pub rust_codegen_units: Option<u32>,
+ pub rust_codegen_units_std: Option<u32>,
+ pub rust_debug_assertions: bool,
+ pub rust_debug_assertions_std: bool,
+ pub rust_overflow_checks: bool,
+ pub rust_overflow_checks_std: bool,
+ pub rust_debug_logging: bool,
+ pub rust_debuginfo_level_rustc: u32,
+ pub rust_debuginfo_level_std: u32,
+ pub rust_debuginfo_level_tools: u32,
+ pub rust_debuginfo_level_tests: u32,
+ pub rust_split_debuginfo: SplitDebuginfo,
+ pub rust_rpath: bool,
+ pub rustc_parallel: bool,
+ pub rustc_default_linker: Option<String>,
+ pub rust_optimize_tests: bool,
+ pub rust_dist_src: bool,
+ pub rust_codegen_backends: Vec<Interned<String>>,
+ pub rust_verify_llvm_ir: bool,
+ pub rust_thin_lto_import_instr_limit: Option<u32>,
+ pub rust_remap_debuginfo: bool,
+ pub rust_new_symbol_mangling: Option<bool>,
+ pub rust_profile_use: Option<String>,
+ pub rust_profile_generate: Option<String>,
+ pub llvm_profile_use: Option<String>,
+ pub llvm_profile_generate: bool,
+ pub llvm_libunwind_default: Option<LlvmLibunwind>,
+
+ pub build: TargetSelection,
+ pub hosts: Vec<TargetSelection>,
+ pub targets: Vec<TargetSelection>,
+ pub local_rebuild: bool,
+ pub jemalloc: bool,
+ pub control_flow_guard: bool,
+
+ // dist misc
+ pub dist_sign_folder: Option<PathBuf>,
+ pub dist_upload_addr: Option<String>,
+ pub dist_compression_formats: Option<Vec<String>>,
+
+ // libstd features
+ pub backtrace: bool, // support for RUST_BACKTRACE
+
+ // misc
+ pub low_priority: bool,
+ pub channel: String,
+ pub description: Option<String>,
+ pub verbose_tests: bool,
+ pub save_toolstates: Option<PathBuf>,
+ pub print_step_timings: bool,
+ pub print_step_rusage: bool,
+ pub missing_tools: bool,
+
+ // Fallback musl-root for all targets
+ pub musl_root: Option<PathBuf>,
+ pub prefix: Option<PathBuf>,
+ pub sysconfdir: Option<PathBuf>,
+ pub datadir: Option<PathBuf>,
+ pub docdir: Option<PathBuf>,
+ pub bindir: PathBuf,
+ pub libdir: Option<PathBuf>,
+ pub mandir: Option<PathBuf>,
+ pub codegen_tests: bool,
+ pub nodejs: Option<PathBuf>,
+ pub npm: Option<PathBuf>,
+ pub gdb: Option<PathBuf>,
+ pub python: Option<PathBuf>,
+ pub cargo_native_static: bool,
+ pub configure_args: Vec<String>,
+
+ // These are either the stage0 downloaded binaries or the locally installed ones.
+ pub initial_cargo: PathBuf,
+ pub initial_rustc: PathBuf,
+ #[cfg(not(test))]
+ initial_rustfmt: RefCell<RustfmtState>,
+ #[cfg(test)]
+ pub initial_rustfmt: RefCell<RustfmtState>,
+ pub out: PathBuf,
+}
+
+#[derive(Default, Deserialize)]
+#[cfg_attr(test, derive(Clone))]
+pub struct Stage0Metadata {
+ pub config: Stage0Config,
+ pub checksums_sha256: HashMap<String, String>,
+ pub rustfmt: Option<RustfmtMetadata>,
+}
+#[derive(Default, Deserialize)]
+#[cfg_attr(test, derive(Clone))]
+pub struct Stage0Config {
+ pub dist_server: String,
+ pub artifacts_server: String,
+ pub artifacts_with_llvm_assertions_server: String,
+ pub git_merge_commit_email: String,
+ pub nightly_branch: String,
+}
+#[derive(Default, Deserialize)]
+#[cfg_attr(test, derive(Clone))]
+pub struct RustfmtMetadata {
+ pub date: String,
+ pub version: String,
+}
+
+#[derive(Clone, Debug)]
+pub enum RustfmtState {
+ SystemToolchain(PathBuf),
+ Downloaded(PathBuf),
+ Unavailable,
+ LazyEvaluated,
+}
+
+impl Default for RustfmtState {
+ fn default() -> Self {
+ RustfmtState::LazyEvaluated
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum LlvmLibunwind {
+ No,
+ InTree,
+ System,
+}
+
+impl Default for LlvmLibunwind {
+ fn default() -> Self {
+ Self::No
+ }
+}
+
+impl FromStr for LlvmLibunwind {
+ type Err = String;
+
+ fn from_str(value: &str) -> Result<Self, Self::Err> {
+ match value {
+ "no" => Ok(Self::No),
+ "in-tree" => Ok(Self::InTree),
+ "system" => Ok(Self::System),
+ invalid => Err(format!("Invalid value '{}' for rust.llvm-libunwind config.", invalid)),
+ }
+ }
+}
+
+#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum SplitDebuginfo {
+ Packed,
+ Unpacked,
+ Off,
+}
+
+impl Default for SplitDebuginfo {
+ fn default() -> Self {
+ SplitDebuginfo::Off
+ }
+}
+
+impl std::str::FromStr for SplitDebuginfo {
+ type Err = ();
+
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
+ match s {
+ "packed" => Ok(SplitDebuginfo::Packed),
+ "unpacked" => Ok(SplitDebuginfo::Unpacked),
+ "off" => Ok(SplitDebuginfo::Off),
+ _ => Err(()),
+ }
+ }
+}
+
+impl SplitDebuginfo {
+ /// Returns the default `-Csplit-debuginfo` value for the current target. See the comment for
+ /// `rust.split-debuginfo` in `config.toml.example`.
+ fn default_for_platform(target: &str) -> Self {
+ if target.contains("apple") {
+ SplitDebuginfo::Unpacked
+ } else if target.contains("windows") {
+ SplitDebuginfo::Packed
+ } else {
+ SplitDebuginfo::Off
+ }
+ }
+}
+
+#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct TargetSelection {
+ pub triple: Interned<String>,
+ file: Option<Interned<String>>,
+}
+
+impl TargetSelection {
+ pub fn from_user(selection: &str) -> Self {
+ let path = Path::new(selection);
+
+ let (triple, file) = if path.exists() {
+ let triple = path
+ .file_stem()
+ .expect("Target specification file has no file stem")
+ .to_str()
+ .expect("Target specification file stem is not UTF-8");
+
+ (triple, Some(selection))
+ } else {
+ (selection, None)
+ };
+
+ let triple = INTERNER.intern_str(triple);
+ let file = file.map(|f| INTERNER.intern_str(f));
+
+ Self { triple, file }
+ }
+
+ pub fn rustc_target_arg(&self) -> &str {
+ self.file.as_ref().unwrap_or(&self.triple)
+ }
+
+ pub fn contains(&self, needle: &str) -> bool {
+ self.triple.contains(needle)
+ }
+
+ pub fn starts_with(&self, needle: &str) -> bool {
+ self.triple.starts_with(needle)
+ }
+
+ pub fn ends_with(&self, needle: &str) -> bool {
+ self.triple.ends_with(needle)
+ }
+}
+
+impl fmt::Display for TargetSelection {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ write!(f, "{}", self.triple)?;
+ if let Some(file) = self.file {
+ write!(f, "({})", file)?;
+ }
+ Ok(())
+ }
+}
+
+impl fmt::Debug for TargetSelection {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ write!(f, "{}", self)
+ }
+}
+
+impl PartialEq<&str> for TargetSelection {
+ fn eq(&self, other: &&str) -> bool {
+ self.triple == *other
+ }
+}
+
+/// Per-target configuration stored in the global configuration structure.
+#[derive(Default)]
+#[cfg_attr(test, derive(Clone))]
+pub struct Target {
+ /// Some(path to llvm-config) if using an external LLVM.
+ pub llvm_config: Option<PathBuf>,
+ /// Some(path to FileCheck) if one was specified.
+ pub llvm_filecheck: Option<PathBuf>,
+ pub llvm_libunwind: Option<LlvmLibunwind>,
+ pub cc: Option<PathBuf>,
+ pub cxx: Option<PathBuf>,
+ pub ar: Option<PathBuf>,
+ pub ranlib: Option<PathBuf>,
+ pub default_linker: Option<PathBuf>,
+ pub linker: Option<PathBuf>,
+ pub ndk: Option<PathBuf>,
+ pub sanitizers: Option<bool>,
+ pub profiler: Option<bool>,
+ pub crt_static: Option<bool>,
+ pub musl_root: Option<PathBuf>,
+ pub musl_libdir: Option<PathBuf>,
+ pub wasi_root: Option<PathBuf>,
+ pub qemu_rootfs: Option<PathBuf>,
+ pub no_std: bool,
+}
+
+impl Target {
+ pub fn from_triple(triple: &str) -> Self {
+ let mut target: Self = Default::default();
+ if triple.contains("-none")
+ || triple.contains("nvptx")
+ || triple.contains("switch")
+ || triple.contains("-uefi")
+ {
+ target.no_std = true;
+ }
+ target
+ }
+}
+/// Structure of the `config.toml` file that configuration is read from.
+///
+/// This structure uses `Decodable` to automatically decode a TOML configuration
+/// file into this format, and then this is traversed and written into the above
+/// `Config` structure.
+#[derive(Deserialize, Default)]
+#[serde(deny_unknown_fields, rename_all = "kebab-case")]
+struct TomlConfig {
+ changelog_seen: Option<usize>,
+ build: Option<Build>,
+ install: Option<Install>,
+ llvm: Option<Llvm>,
+ rust: Option<Rust>,
+ target: Option<HashMap<String, TomlTarget>>,
+ dist: Option<Dist>,
+ profile: Option<String>,
+}
+
+trait Merge {
+ fn merge(&mut self, other: Self);
+}
+
+impl Merge for TomlConfig {
+ fn merge(
+ &mut self,
+ TomlConfig { build, install, llvm, rust, dist, target, profile: _, changelog_seen: _ }: Self,
+ ) {
+ fn do_merge<T: Merge>(x: &mut Option<T>, y: Option<T>) {
+ if let Some(new) = y {
+ if let Some(original) = x {
+ original.merge(new);
+ } else {
+ *x = Some(new);
+ }
+ }
+ }
+ do_merge(&mut self.build, build);
+ do_merge(&mut self.install, install);
+ do_merge(&mut self.llvm, llvm);
+ do_merge(&mut self.rust, rust);
+ do_merge(&mut self.dist, dist);
+ assert!(target.is_none(), "merging target-specific config is not currently supported");
+ }
+}
+
+// We are using a decl macro instead of a derive proc macro here to reduce the compile time of
+// rustbuild.
+macro_rules! define_config {
+ ($(#[$attr:meta])* struct $name:ident {
+ $($field:ident: Option<$field_ty:ty> = $field_key:literal,)*
+ }) => {
+ $(#[$attr])*
+ struct $name {
+ $($field: Option<$field_ty>,)*
+ }
+
+ impl Merge for $name {
+ fn merge(&mut self, other: Self) {
+ $(
+ if !self.$field.is_some() {
+ self.$field = other.$field;
+ }
+ )*
+ }
+ }
+
+ // The following is a trimmed version of what serde_derive generates. All parts not relevant
+ // for toml deserialization have been removed. This reduces the binary size and improves
+ // compile time of rustbuild.
+ impl<'de> Deserialize<'de> for $name {
+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+ where
+ D: Deserializer<'de>,
+ {
+ struct Field;
+ impl<'de> serde::de::Visitor<'de> for Field {
+ type Value = $name;
+ fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.write_str(concat!("struct ", stringify!($name)))
+ }
+
+ #[inline]
+ fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
+ where
+ A: serde::de::MapAccess<'de>,
+ {
+ $(let mut $field: Option<$field_ty> = None;)*
+ while let Some(key) =
+ match serde::de::MapAccess::next_key::<String>(&mut map) {
+ Ok(val) => val,
+ Err(err) => {
+ return Err(err);
+ }
+ }
+ {
+ match &*key {
+ $($field_key => {
+ if $field.is_some() {
+ return Err(<A::Error as serde::de::Error>::duplicate_field(
+ $field_key,
+ ));
+ }
+ $field = match serde::de::MapAccess::next_value::<$field_ty>(
+ &mut map,
+ ) {
+ Ok(val) => Some(val),
+ Err(err) => {
+ return Err(err);
+ }
+ };
+ })*
+ key => {
+ return Err(serde::de::Error::unknown_field(key, FIELDS));
+ }
+ }
+ }
+ Ok($name { $($field),* })
+ }
+ }
+ const FIELDS: &'static [&'static str] = &[
+ $($field_key,)*
+ ];
+ Deserializer::deserialize_struct(
+ deserializer,
+ stringify!($name),
+ FIELDS,
+ Field,
+ )
+ }
+ }
+ }
+}
+
+define_config! {
+ /// TOML representation of various global build decisions.
+ #[derive(Default)]
+ struct Build {
+ build: Option<String> = "build",
+ host: Option<Vec<String>> = "host",
+ target: Option<Vec<String>> = "target",
+ build_dir: Option<String> = "build-dir",
+ cargo: Option<String> = "cargo",
+ rustc: Option<String> = "rustc",
+ rustfmt: Option<PathBuf> = "rustfmt",
+ docs: Option<bool> = "docs",
+ compiler_docs: Option<bool> = "compiler-docs",
+ docs_minification: Option<bool> = "docs-minification",
+ submodules: Option<bool> = "submodules",
+ gdb: Option<String> = "gdb",
+ nodejs: Option<String> = "nodejs",
+ npm: Option<String> = "npm",
+ python: Option<String> = "python",
+ locked_deps: Option<bool> = "locked-deps",
+ vendor: Option<bool> = "vendor",
+ full_bootstrap: Option<bool> = "full-bootstrap",
+ extended: Option<bool> = "extended",
+ tools: Option<HashSet<String>> = "tools",
+ verbose: Option<usize> = "verbose",
+ sanitizers: Option<bool> = "sanitizers",
+ profiler: Option<bool> = "profiler",
+ cargo_native_static: Option<bool> = "cargo-native-static",
+ low_priority: Option<bool> = "low-priority",
+ configure_args: Option<Vec<String>> = "configure-args",
+ local_rebuild: Option<bool> = "local-rebuild",
+ print_step_timings: Option<bool> = "print-step-timings",
+ print_step_rusage: Option<bool> = "print-step-rusage",
+ check_stage: Option<u32> = "check-stage",
+ doc_stage: Option<u32> = "doc-stage",
+ build_stage: Option<u32> = "build-stage",
+ test_stage: Option<u32> = "test-stage",
+ install_stage: Option<u32> = "install-stage",
+ dist_stage: Option<u32> = "dist-stage",
+ bench_stage: Option<u32> = "bench-stage",
+ patch_binaries_for_nix: Option<bool> = "patch-binaries-for-nix",
+ metrics: Option<bool> = "metrics",
+ }
+}
+
+define_config! {
+ /// TOML representation of various global install decisions.
+ struct Install {
+ prefix: Option<String> = "prefix",
+ sysconfdir: Option<String> = "sysconfdir",
+ docdir: Option<String> = "docdir",
+ bindir: Option<String> = "bindir",
+ libdir: Option<String> = "libdir",
+ mandir: Option<String> = "mandir",
+ datadir: Option<String> = "datadir",
+ }
+}
+
+define_config! {
+ /// TOML representation of how the LLVM build is configured.
+ struct Llvm {
+ skip_rebuild: Option<bool> = "skip-rebuild",
+ optimize: Option<bool> = "optimize",
+ thin_lto: Option<bool> = "thin-lto",
+ release_debuginfo: Option<bool> = "release-debuginfo",
+ assertions: Option<bool> = "assertions",
+ tests: Option<bool> = "tests",
+ plugins: Option<bool> = "plugins",
+ ccache: Option<StringOrBool> = "ccache",
+ version_check: Option<bool> = "version-check",
+ static_libstdcpp: Option<bool> = "static-libstdcpp",
+ ninja: Option<bool> = "ninja",
+ targets: Option<String> = "targets",
+ experimental_targets: Option<String> = "experimental-targets",
+ link_jobs: Option<u32> = "link-jobs",
+ link_shared: Option<bool> = "link-shared",
+ version_suffix: Option<String> = "version-suffix",
+ clang_cl: Option<String> = "clang-cl",
+ cflags: Option<String> = "cflags",
+ cxxflags: Option<String> = "cxxflags",
+ ldflags: Option<String> = "ldflags",
+ use_libcxx: Option<bool> = "use-libcxx",
+ use_linker: Option<String> = "use-linker",
+ allow_old_toolchain: Option<bool> = "allow-old-toolchain",
+ polly: Option<bool> = "polly",
+ clang: Option<bool> = "clang",
+ download_ci_llvm: Option<StringOrBool> = "download-ci-llvm",
+ build_config: Option<HashMap<String, String>> = "build-config",
+ }
+}
+
+define_config! {
+ struct Dist {
+ sign_folder: Option<String> = "sign-folder",
+ gpg_password_file: Option<String> = "gpg-password-file",
+ upload_addr: Option<String> = "upload-addr",
+ src_tarball: Option<bool> = "src-tarball",
+ missing_tools: Option<bool> = "missing-tools",
+ compression_formats: Option<Vec<String>> = "compression-formats",
+ }
+}
+
+#[derive(Deserialize)]
+#[serde(untagged)]
+enum StringOrBool {
+ String(String),
+ Bool(bool),
+}
+
+impl Default for StringOrBool {
+ fn default() -> StringOrBool {
+ StringOrBool::Bool(false)
+ }
+}
+
+define_config! {
+ /// TOML representation of how the Rust build is configured.
+ struct Rust {
+ optimize: Option<bool> = "optimize",
+ debug: Option<bool> = "debug",
+ codegen_units: Option<u32> = "codegen-units",
+ codegen_units_std: Option<u32> = "codegen-units-std",
+ debug_assertions: Option<bool> = "debug-assertions",
+ debug_assertions_std: Option<bool> = "debug-assertions-std",
+ overflow_checks: Option<bool> = "overflow-checks",
+ overflow_checks_std: Option<bool> = "overflow-checks-std",
+ debug_logging: Option<bool> = "debug-logging",
+ debuginfo_level: Option<u32> = "debuginfo-level",
+ debuginfo_level_rustc: Option<u32> = "debuginfo-level-rustc",
+ debuginfo_level_std: Option<u32> = "debuginfo-level-std",
+ debuginfo_level_tools: Option<u32> = "debuginfo-level-tools",
+ debuginfo_level_tests: Option<u32> = "debuginfo-level-tests",
+ split_debuginfo: Option<String> = "split-debuginfo",
+ run_dsymutil: Option<bool> = "run-dsymutil",
+ backtrace: Option<bool> = "backtrace",
+ incremental: Option<bool> = "incremental",
+ parallel_compiler: Option<bool> = "parallel-compiler",
+ default_linker: Option<String> = "default-linker",
+ channel: Option<String> = "channel",
+ description: Option<String> = "description",
+ musl_root: Option<String> = "musl-root",
+ rpath: Option<bool> = "rpath",
+ verbose_tests: Option<bool> = "verbose-tests",
+ optimize_tests: Option<bool> = "optimize-tests",
+ codegen_tests: Option<bool> = "codegen-tests",
+ ignore_git: Option<bool> = "ignore-git",
+ dist_src: Option<bool> = "dist-src",
+ save_toolstates: Option<String> = "save-toolstates",
+ codegen_backends: Option<Vec<String>> = "codegen-backends",
+ lld: Option<bool> = "lld",
+ use_lld: Option<bool> = "use-lld",
+ llvm_tools: Option<bool> = "llvm-tools",
+ deny_warnings: Option<bool> = "deny-warnings",
+ backtrace_on_ice: Option<bool> = "backtrace-on-ice",
+ verify_llvm_ir: Option<bool> = "verify-llvm-ir",
+ thin_lto_import_instr_limit: Option<u32> = "thin-lto-import-instr-limit",
+ remap_debuginfo: Option<bool> = "remap-debuginfo",
+ jemalloc: Option<bool> = "jemalloc",
+ test_compare_mode: Option<bool> = "test-compare-mode",
+ llvm_libunwind: Option<String> = "llvm-libunwind",
+ control_flow_guard: Option<bool> = "control-flow-guard",
+ new_symbol_mangling: Option<bool> = "new-symbol-mangling",
+ profile_generate: Option<String> = "profile-generate",
+ profile_use: Option<String> = "profile-use",
+ // ignored; this is set from an env var set by bootstrap.py
+ download_rustc: Option<StringOrBool> = "download-rustc",
+ }
+}
+
+define_config! {
+ /// TOML representation of how each build target is configured.
+ struct TomlTarget {
+ cc: Option<String> = "cc",
+ cxx: Option<String> = "cxx",
+ ar: Option<String> = "ar",
+ ranlib: Option<String> = "ranlib",
+ default_linker: Option<PathBuf> = "default-linker",
+ linker: Option<String> = "linker",
+ llvm_config: Option<String> = "llvm-config",
+ llvm_filecheck: Option<String> = "llvm-filecheck",
+ llvm_libunwind: Option<String> = "llvm-libunwind",
+ android_ndk: Option<String> = "android-ndk",
+ sanitizers: Option<bool> = "sanitizers",
+ profiler: Option<bool> = "profiler",
+ crt_static: Option<bool> = "crt-static",
+ musl_root: Option<String> = "musl-root",
+ musl_libdir: Option<String> = "musl-libdir",
+ wasi_root: Option<String> = "wasi-root",
+ qemu_rootfs: Option<String> = "qemu-rootfs",
+ no_std: Option<bool> = "no-std",
+ }
+}
+
+impl Config {
+ pub fn default_opts() -> Config {
+ let mut config = Config::default();
+ config.llvm_optimize = true;
+ config.ninja_in_file = true;
+ config.llvm_version_check = true;
+ config.llvm_static_stdcpp = true;
+ config.backtrace = true;
+ config.rust_optimize = true;
+ config.rust_optimize_tests = true;
+ config.submodules = None;
+ config.docs = true;
+ config.docs_minification = true;
+ config.rust_rpath = true;
+ config.channel = "dev".to_string();
+ config.codegen_tests = true;
+ config.rust_dist_src = true;
+ config.rust_codegen_backends = vec![INTERNER.intern_str("llvm")];
+ config.deny_warnings = true;
+ config.bindir = "bin".into();
+
+ // set by build.rs
+ config.build = TargetSelection::from_user(&env!("BUILD_TRIPLE"));
+ let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+ // Undo `src/bootstrap`
+ config.src = manifest_dir.parent().unwrap().parent().unwrap().to_owned();
+ config.out = PathBuf::from("build");
+
+ config.initial_cargo = PathBuf::from(env!("CARGO"));
+ config.initial_rustc = PathBuf::from(env!("RUSTC"));
+
+ config
+ }
+
+ pub fn parse(args: &[String]) -> Config {
+ let flags = Flags::parse(&args);
+
+ let mut config = Config::default_opts();
+ config.exclude = flags.exclude.into_iter().map(|path| TaskPath::parse(path)).collect();
+ config.include_default_paths = flags.include_default_paths;
+ config.rustc_error_format = flags.rustc_error_format;
+ config.json_output = flags.json_output;
+ config.on_fail = flags.on_fail;
+ config.jobs = flags.jobs.map(threads_from_config);
+ config.cmd = flags.cmd;
+ config.incremental = flags.incremental;
+ config.dry_run = flags.dry_run;
+ config.keep_stage = flags.keep_stage;
+ config.keep_stage_std = flags.keep_stage_std;
+ config.color = flags.color;
+ if let Some(value) = flags.deny_warnings {
+ config.deny_warnings = value;
+ }
+ config.llvm_profile_use = flags.llvm_profile_use;
+ config.llvm_profile_generate = flags.llvm_profile_generate;
+
+ let stage0_json = t!(std::fs::read(&config.src.join("src").join("stage0.json")));
+ config.stage0_metadata = t!(serde_json::from_slice::<Stage0Metadata>(&stage0_json));
+
+ #[cfg(test)]
+ let get_toml = |_| TomlConfig::default();
+ #[cfg(not(test))]
+ let get_toml = |file: &Path| {
+ let contents =
+ t!(fs::read_to_string(file), format!("config file {} not found", file.display()));
+ // Deserialize to Value and then TomlConfig to prevent the Deserialize impl of
+ // TomlConfig and sub types to be monomorphized 5x by toml.
+ match toml::from_str(&contents)
+ .and_then(|table: toml::Value| TomlConfig::deserialize(table))
+ {
+ Ok(table) => table,
+ Err(err) => {
+ eprintln!("failed to parse TOML configuration '{}': {}", file.display(), err);
+ crate::detail_exit(2);
+ }
+ }
+ };
+
+ // Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory.
+ let toml_path = flags
+ .config
+ .clone()
+ .or_else(|| env::var_os("RUST_BOOTSTRAP_CONFIG").map(PathBuf::from));
+ let using_default_path = toml_path.is_none();
+ let mut toml_path = toml_path.unwrap_or_else(|| PathBuf::from("config.toml"));
+ if using_default_path && !toml_path.exists() {
+ toml_path = config.src.join(toml_path);
+ }
+
+ // Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path,
+ // but not if `config.toml` hasn't been created.
+ let mut toml = if !using_default_path || toml_path.exists() {
+ get_toml(&toml_path)
+ } else {
+ TomlConfig::default()
+ };
+
+ if let Some(include) = &toml.profile {
+ let mut include_path = config.src.clone();
+ include_path.push("src");
+ include_path.push("bootstrap");
+ include_path.push("defaults");
+ include_path.push(format!("config.{}.toml", include));
+ let included_toml = get_toml(&include_path);
+ toml.merge(included_toml);
+ }
+
+ config.changelog_seen = toml.changelog_seen;
+ config.config = toml_path;
+
+ let build = toml.build.unwrap_or_default();
+
+ set(&mut config.initial_rustc, build.rustc.map(PathBuf::from));
+ set(&mut config.out, flags.build_dir.or_else(|| build.build_dir.map(PathBuf::from)));
+ // NOTE: Bootstrap spawns various commands with different working directories.
+ // To avoid writing to random places on the file system, `config.out` needs to be an absolute path.
+ if !config.out.is_absolute() {
+ // `canonicalize` requires the path to already exist. Use our vendored copy of `absolute` instead.
+ config.out = crate::util::absolute(&config.out);
+ }
+
+ if config.dry_run {
+ let dir = config.out.join("tmp-dry-run");
+ t!(fs::create_dir_all(&dir));
+ config.out = dir;
+ }
+
+ config.hosts = if let Some(arg_host) = flags.host {
+ arg_host
+ } else if let Some(file_host) = build.host {
+ file_host.iter().map(|h| TargetSelection::from_user(h)).collect()
+ } else {
+ vec![config.build]
+ };
+ config.targets = if let Some(arg_target) = flags.target {
+ arg_target
+ } else if let Some(file_target) = build.target {
+ file_target.iter().map(|h| TargetSelection::from_user(h)).collect()
+ } else {
+ // If target is *not* configured, then default to the host
+ // toolchains.
+ config.hosts.clone()
+ };
+
+ config.nodejs = build.nodejs.map(PathBuf::from);
+ config.npm = build.npm.map(PathBuf::from);
+ config.gdb = build.gdb.map(PathBuf::from);
+ config.python = build.python.map(PathBuf::from);
+ config.submodules = build.submodules;
+ set(&mut config.low_priority, build.low_priority);
+ set(&mut config.compiler_docs, build.compiler_docs);
+ set(&mut config.docs_minification, build.docs_minification);
+ set(&mut config.docs, build.docs);
+ set(&mut config.locked_deps, build.locked_deps);
+ set(&mut config.vendor, build.vendor);
+ set(&mut config.full_bootstrap, build.full_bootstrap);
+ set(&mut config.extended, build.extended);
+ config.tools = build.tools;
+ set(&mut config.verbose, build.verbose);
+ set(&mut config.sanitizers, build.sanitizers);
+ set(&mut config.profiler, build.profiler);
+ set(&mut config.cargo_native_static, build.cargo_native_static);
+ set(&mut config.configure_args, build.configure_args);
+ set(&mut config.local_rebuild, build.local_rebuild);
+ set(&mut config.print_step_timings, build.print_step_timings);
+ set(&mut config.print_step_rusage, build.print_step_rusage);
+ set(&mut config.patch_binaries_for_nix, build.patch_binaries_for_nix);
+
+ config.verbose = cmp::max(config.verbose, flags.verbose);
+
+ if let Some(install) = toml.install {
+ config.prefix = install.prefix.map(PathBuf::from);
+ config.sysconfdir = install.sysconfdir.map(PathBuf::from);
+ config.datadir = install.datadir.map(PathBuf::from);
+ config.docdir = install.docdir.map(PathBuf::from);
+ set(&mut config.bindir, install.bindir.map(PathBuf::from));
+ config.libdir = install.libdir.map(PathBuf::from);
+ config.mandir = install.mandir.map(PathBuf::from);
+ }
+
+ // We want the llvm-skip-rebuild flag to take precedence over the
+ // skip-rebuild config.toml option so we store it separately
+ // so that we can infer the right value
+ let mut llvm_skip_rebuild = flags.llvm_skip_rebuild;
+
+ // Store off these values as options because if they're not provided
+ // we'll infer default values for them later
+ let mut llvm_assertions = None;
+ let mut llvm_tests = None;
+ let mut llvm_plugins = None;
+ let mut debug = None;
+ let mut debug_assertions = None;
+ let mut debug_assertions_std = None;
+ let mut overflow_checks = None;
+ let mut overflow_checks_std = None;
+ let mut debug_logging = None;
+ let mut debuginfo_level = None;
+ let mut debuginfo_level_rustc = None;
+ let mut debuginfo_level_std = None;
+ let mut debuginfo_level_tools = None;
+ let mut debuginfo_level_tests = None;
+ let mut optimize = None;
+ let mut ignore_git = None;
+
+ if let Some(llvm) = toml.llvm {
+ match llvm.ccache {
+ Some(StringOrBool::String(ref s)) => config.ccache = Some(s.to_string()),
+ Some(StringOrBool::Bool(true)) => {
+ config.ccache = Some("ccache".to_string());
+ }
+ Some(StringOrBool::Bool(false)) | None => {}
+ }
+ set(&mut config.ninja_in_file, llvm.ninja);
+ llvm_assertions = llvm.assertions;
+ llvm_tests = llvm.tests;
+ llvm_plugins = llvm.plugins;
+ llvm_skip_rebuild = llvm_skip_rebuild.or(llvm.skip_rebuild);
+ set(&mut config.llvm_optimize, llvm.optimize);
+ set(&mut config.llvm_thin_lto, llvm.thin_lto);
+ set(&mut config.llvm_release_debuginfo, llvm.release_debuginfo);
+ set(&mut config.llvm_version_check, llvm.version_check);
+ set(&mut config.llvm_static_stdcpp, llvm.static_libstdcpp);
+ if let Some(v) = llvm.link_shared {
+ config.llvm_link_shared.set(Some(v));
+ }
+ config.llvm_targets = llvm.targets.clone();
+ config.llvm_experimental_targets = llvm.experimental_targets.clone();
+ config.llvm_link_jobs = llvm.link_jobs;
+ config.llvm_version_suffix = llvm.version_suffix.clone();
+ config.llvm_clang_cl = llvm.clang_cl.clone();
+
+ config.llvm_cflags = llvm.cflags.clone();
+ config.llvm_cxxflags = llvm.cxxflags.clone();
+ config.llvm_ldflags = llvm.ldflags.clone();
+ set(&mut config.llvm_use_libcxx, llvm.use_libcxx);
+ config.llvm_use_linker = llvm.use_linker.clone();
+ config.llvm_allow_old_toolchain = llvm.allow_old_toolchain.unwrap_or(false);
+ config.llvm_polly = llvm.polly.unwrap_or(false);
+ config.llvm_clang = llvm.clang.unwrap_or(false);
+ config.llvm_build_config = llvm.build_config.clone().unwrap_or(Default::default());
+ config.llvm_from_ci = match llvm.download_ci_llvm {
+ Some(StringOrBool::String(s)) => {
+ assert!(s == "if-available", "unknown option `{}` for download-ci-llvm", s);
+ // This is currently all tier 1 targets and tier 2 targets with host tools
+ // (since others may not have CI artifacts)
+ // https://doc.rust-lang.org/rustc/platform-support.html#tier-1
+ // FIXME: this is duplicated in bootstrap.py
+ let supported_platforms = [
+ // tier 1
+ "aarch64-unknown-linux-gnu",
+ "i686-pc-windows-gnu",
+ "i686-pc-windows-msvc",
+ "i686-unknown-linux-gnu",
+ "x86_64-unknown-linux-gnu",
+ "x86_64-apple-darwin",
+ "x86_64-pc-windows-gnu",
+ "x86_64-pc-windows-msvc",
+ // tier 2 with host tools
+ "aarch64-apple-darwin",
+ "aarch64-pc-windows-msvc",
+ "aarch64-unknown-linux-musl",
+ "arm-unknown-linux-gnueabi",
+ "arm-unknown-linux-gnueabihf",
+ "armv7-unknown-linux-gnueabihf",
+ "mips-unknown-linux-gnu",
+ "mips64-unknown-linux-gnuabi64",
+ "mips64el-unknown-linux-gnuabi64",
+ "mipsel-unknown-linux-gnu",
+ "powerpc-unknown-linux-gnu",
+ "powerpc64-unknown-linux-gnu",
+ "powerpc64le-unknown-linux-gnu",
+ "riscv64gc-unknown-linux-gnu",
+ "s390x-unknown-linux-gnu",
+ "x86_64-unknown-freebsd",
+ "x86_64-unknown-illumos",
+ "x86_64-unknown-linux-musl",
+ "x86_64-unknown-netbsd",
+ ];
+ supported_platforms.contains(&&*config.build.triple)
+ }
+ Some(StringOrBool::Bool(b)) => b,
+ None => false,
+ };
+
+ if config.llvm_from_ci {
+ // None of the LLVM options, except assertions, are supported
+ // when using downloaded LLVM. We could just ignore these but
+ // that's potentially confusing, so force them to not be
+ // explicitly set. The defaults and CI defaults don't
+ // necessarily match but forcing people to match (somewhat
+ // arbitrary) CI configuration locally seems bad/hard.
+ check_ci_llvm!(llvm.optimize);
+ check_ci_llvm!(llvm.thin_lto);
+ check_ci_llvm!(llvm.release_debuginfo);
+ // CI-built LLVM can be either dynamic or static. We won't know until we download it.
+ check_ci_llvm!(llvm.link_shared);
+ check_ci_llvm!(llvm.static_libstdcpp);
+ check_ci_llvm!(llvm.targets);
+ check_ci_llvm!(llvm.experimental_targets);
+ check_ci_llvm!(llvm.link_jobs);
+ check_ci_llvm!(llvm.clang_cl);
+ check_ci_llvm!(llvm.version_suffix);
+ check_ci_llvm!(llvm.cflags);
+ check_ci_llvm!(llvm.cxxflags);
+ check_ci_llvm!(llvm.ldflags);
+ check_ci_llvm!(llvm.use_libcxx);
+ check_ci_llvm!(llvm.use_linker);
+ check_ci_llvm!(llvm.allow_old_toolchain);
+ check_ci_llvm!(llvm.polly);
+ check_ci_llvm!(llvm.clang);
+ check_ci_llvm!(llvm.build_config);
+ check_ci_llvm!(llvm.plugins);
+ }
+
+ // NOTE: can never be hit when downloading from CI, since we call `check_ci_llvm!(thin_lto)` above.
+ if config.llvm_thin_lto && llvm.link_shared.is_none() {
+ // If we're building with ThinLTO on, by default we want to link
+ // to LLVM shared, to avoid re-doing ThinLTO (which happens in
+ // the link step) with each stage.
+ config.llvm_link_shared.set(Some(true));
+ }
+ }
+
+ if let Some(rust) = toml.rust {
+ debug = rust.debug;
+ debug_assertions = rust.debug_assertions;
+ debug_assertions_std = rust.debug_assertions_std;
+ overflow_checks = rust.overflow_checks;
+ overflow_checks_std = rust.overflow_checks_std;
+ debug_logging = rust.debug_logging;
+ debuginfo_level = rust.debuginfo_level;
+ debuginfo_level_rustc = rust.debuginfo_level_rustc;
+ debuginfo_level_std = rust.debuginfo_level_std;
+ debuginfo_level_tools = rust.debuginfo_level_tools;
+ debuginfo_level_tests = rust.debuginfo_level_tests;
+ config.rust_split_debuginfo = rust
+ .split_debuginfo
+ .as_deref()
+ .map(SplitDebuginfo::from_str)
+ .map(|v| v.expect("invalid value for rust.split_debuginfo"))
+ .unwrap_or(SplitDebuginfo::default_for_platform(&config.build.triple));
+ optimize = rust.optimize;
+ ignore_git = rust.ignore_git;
+ config.rust_new_symbol_mangling = rust.new_symbol_mangling;
+ set(&mut config.rust_optimize_tests, rust.optimize_tests);
+ set(&mut config.codegen_tests, rust.codegen_tests);
+ set(&mut config.rust_rpath, rust.rpath);
+ set(&mut config.jemalloc, rust.jemalloc);
+ set(&mut config.test_compare_mode, rust.test_compare_mode);
+ set(&mut config.backtrace, rust.backtrace);
+ set(&mut config.channel, rust.channel);
+ config.description = rust.description;
+ set(&mut config.rust_dist_src, rust.dist_src);
+ set(&mut config.verbose_tests, rust.verbose_tests);
+ // in the case "false" is set explicitly, do not overwrite the command line args
+ if let Some(true) = rust.incremental {
+ config.incremental = true;
+ }
+ set(&mut config.use_lld, rust.use_lld);
+ set(&mut config.lld_enabled, rust.lld);
+ set(&mut config.llvm_tools_enabled, rust.llvm_tools);
+ config.rustc_parallel = rust.parallel_compiler.unwrap_or(false);
+ config.rustc_default_linker = rust.default_linker;
+ config.musl_root = rust.musl_root.map(PathBuf::from);
+ config.save_toolstates = rust.save_toolstates.map(PathBuf::from);
+ set(&mut config.deny_warnings, flags.deny_warnings.or(rust.deny_warnings));
+ set(&mut config.backtrace_on_ice, rust.backtrace_on_ice);
+ set(&mut config.rust_verify_llvm_ir, rust.verify_llvm_ir);
+ config.rust_thin_lto_import_instr_limit = rust.thin_lto_import_instr_limit;
+ set(&mut config.rust_remap_debuginfo, rust.remap_debuginfo);
+ set(&mut config.control_flow_guard, rust.control_flow_guard);
+ config.llvm_libunwind_default = rust
+ .llvm_libunwind
+ .map(|v| v.parse().expect("failed to parse rust.llvm-libunwind"));
+
+ if let Some(ref backends) = rust.codegen_backends {
+ config.rust_codegen_backends =
+ backends.iter().map(|s| INTERNER.intern_str(s)).collect();
+ }
+
+ config.rust_codegen_units = rust.codegen_units.map(threads_from_config);
+ config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config);
+ config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use);
+ config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate);
+ config.download_rustc_commit = download_ci_rustc_commit(&config, rust.download_rustc);
+ } else {
+ config.rust_profile_use = flags.rust_profile_use;
+ config.rust_profile_generate = flags.rust_profile_generate;
+ }
+
+ if let Some(t) = toml.target {
+ for (triple, cfg) in t {
+ let mut target = Target::from_triple(&triple);
+
+ if let Some(ref s) = cfg.llvm_config {
+ target.llvm_config = Some(config.src.join(s));
+ }
+ if let Some(ref s) = cfg.llvm_filecheck {
+ target.llvm_filecheck = Some(config.src.join(s));
+ }
+ target.llvm_libunwind = cfg
+ .llvm_libunwind
+ .as_ref()
+ .map(|v| v.parse().expect("failed to parse rust.llvm-libunwind"));
+ if let Some(ref s) = cfg.android_ndk {
+ target.ndk = Some(config.src.join(s));
+ }
+ if let Some(s) = cfg.no_std {
+ target.no_std = s;
+ }
+ target.cc = cfg.cc.map(PathBuf::from);
+ target.cxx = cfg.cxx.map(PathBuf::from);
+ target.ar = cfg.ar.map(PathBuf::from);
+ target.ranlib = cfg.ranlib.map(PathBuf::from);
+ target.linker = cfg.linker.map(PathBuf::from);
+ target.crt_static = cfg.crt_static;
+ target.musl_root = cfg.musl_root.map(PathBuf::from);
+ target.musl_libdir = cfg.musl_libdir.map(PathBuf::from);
+ target.wasi_root = cfg.wasi_root.map(PathBuf::from);
+ target.qemu_rootfs = cfg.qemu_rootfs.map(PathBuf::from);
+ target.sanitizers = cfg.sanitizers;
+ target.profiler = cfg.profiler;
+
+ config.target_config.insert(TargetSelection::from_user(&triple), target);
+ }
+ }
+
+ if config.llvm_from_ci {
+ let triple = &config.build.triple;
+ let mut build_target = config
+ .target_config
+ .entry(config.build)
+ .or_insert_with(|| Target::from_triple(&triple));
+
+ check_ci_llvm!(build_target.llvm_config);
+ check_ci_llvm!(build_target.llvm_filecheck);
+ let ci_llvm_bin = config.out.join(&*config.build.triple).join("ci-llvm/bin");
+ build_target.llvm_config = Some(ci_llvm_bin.join(exe("llvm-config", config.build)));
+ build_target.llvm_filecheck = Some(ci_llvm_bin.join(exe("FileCheck", config.build)));
+ }
+
+ if let Some(t) = toml.dist {
+ config.dist_sign_folder = t.sign_folder.map(PathBuf::from);
+ config.dist_upload_addr = t.upload_addr;
+ config.dist_compression_formats = t.compression_formats;
+ set(&mut config.rust_dist_src, t.src_tarball);
+ set(&mut config.missing_tools, t.missing_tools);
+ }
+
+ if let Some(r) = build.rustfmt {
+ *config.initial_rustfmt.borrow_mut() = if r.exists() {
+ RustfmtState::SystemToolchain(r)
+ } else {
+ RustfmtState::Unavailable
+ };
+ } else {
+ // If using a system toolchain for bootstrapping, see if that has rustfmt available.
+ let host = config.build;
+ let rustfmt_path = config.initial_rustc.with_file_name(exe("rustfmt", host));
+ let bin_root = config.out.join(host.triple).join("stage0");
+ if !rustfmt_path.starts_with(&bin_root) {
+ // Using a system-provided toolchain; we shouldn't download rustfmt.
+ *config.initial_rustfmt.borrow_mut() = RustfmtState::SystemToolchain(rustfmt_path);
+ }
+ }
+
+ // Now that we've reached the end of our configuration, infer the
+ // default values for all options that we haven't otherwise stored yet.
+
+ config.llvm_skip_rebuild = llvm_skip_rebuild.unwrap_or(false);
+ config.llvm_assertions = llvm_assertions.unwrap_or(false);
+ config.llvm_tests = llvm_tests.unwrap_or(false);
+ config.llvm_plugins = llvm_plugins.unwrap_or(false);
+ config.rust_optimize = optimize.unwrap_or(true);
+
+ let default = debug == Some(true);
+ config.rust_debug_assertions = debug_assertions.unwrap_or(default);
+ config.rust_debug_assertions_std =
+ debug_assertions_std.unwrap_or(config.rust_debug_assertions);
+ config.rust_overflow_checks = overflow_checks.unwrap_or(default);
+ config.rust_overflow_checks_std =
+ overflow_checks_std.unwrap_or(config.rust_overflow_checks);
+
+ config.rust_debug_logging = debug_logging.unwrap_or(config.rust_debug_assertions);
+
+ let with_defaults = |debuginfo_level_specific: Option<u32>| {
+ debuginfo_level_specific.or(debuginfo_level).unwrap_or(if debug == Some(true) {
+ 1
+ } else {
+ 0
+ })
+ };
+ config.rust_debuginfo_level_rustc = with_defaults(debuginfo_level_rustc);
+ config.rust_debuginfo_level_std = with_defaults(debuginfo_level_std);
+ config.rust_debuginfo_level_tools = with_defaults(debuginfo_level_tools);
+ config.rust_debuginfo_level_tests = debuginfo_level_tests.unwrap_or(0);
+
+ let default = config.channel == "dev";
+ config.ignore_git = ignore_git.unwrap_or(default);
+
+ let download_rustc = config.download_rustc_commit.is_some();
+ // See https://github.com/rust-lang/compiler-team/issues/326
+ config.stage = match config.cmd {
+ Subcommand::Check { .. } => flags.stage.or(build.check_stage).unwrap_or(0),
+ // `download-rustc` only has a speed-up for stage2 builds. Default to stage2 unless explicitly overridden.
+ Subcommand::Doc { .. } => {
+ flags.stage.or(build.doc_stage).unwrap_or(if download_rustc { 2 } else { 0 })
+ }
+ Subcommand::Build { .. } => {
+ flags.stage.or(build.build_stage).unwrap_or(if download_rustc { 2 } else { 1 })
+ }
+ Subcommand::Test { .. } => {
+ flags.stage.or(build.test_stage).unwrap_or(if download_rustc { 2 } else { 1 })
+ }
+ Subcommand::Bench { .. } => flags.stage.or(build.bench_stage).unwrap_or(2),
+ Subcommand::Dist { .. } => flags.stage.or(build.dist_stage).unwrap_or(2),
+ Subcommand::Install { .. } => flags.stage.or(build.install_stage).unwrap_or(2),
+ // These are all bootstrap tools, which don't depend on the compiler.
+ // The stage we pass shouldn't matter, but use 0 just in case.
+ Subcommand::Clean { .. }
+ | Subcommand::Clippy { .. }
+ | Subcommand::Fix { .. }
+ | Subcommand::Run { .. }
+ | Subcommand::Setup { .. }
+ | Subcommand::Format { .. } => flags.stage.unwrap_or(0),
+ };
+
+ // CI should always run stage 2 builds, unless it specifically states otherwise
+ #[cfg(not(test))]
+ if flags.stage.is_none() && crate::CiEnv::current() != crate::CiEnv::None {
+ match config.cmd {
+ Subcommand::Test { .. }
+ | Subcommand::Doc { .. }
+ | Subcommand::Build { .. }
+ | Subcommand::Bench { .. }
+ | Subcommand::Dist { .. }
+ | Subcommand::Install { .. } => {
+ assert_eq!(
+ config.stage, 2,
+ "x.py should be run with `--stage 2` on CI, but was run with `--stage {}`",
+ config.stage,
+ );
+ }
+ Subcommand::Clean { .. }
+ | Subcommand::Check { .. }
+ | Subcommand::Clippy { .. }
+ | Subcommand::Fix { .. }
+ | Subcommand::Run { .. }
+ | Subcommand::Setup { .. }
+ | Subcommand::Format { .. } => {}
+ }
+ }
+
+ config
+ }
+
+ /// A git invocation which runs inside the source directory.
+ ///
+ /// Use this rather than `Command::new("git")` in order to support out-of-tree builds.
+ pub(crate) fn git(&self) -> Command {
+ let mut git = Command::new("git");
+ git.current_dir(&self.src);
+ git
+ }
+
+ pub(crate) fn artifact_channel(&self, commit: &str) -> String {
+ let mut channel = self.git();
+ channel.arg("show").arg(format!("{}:src/ci/channel", commit));
+ let channel = output(&mut channel);
+ channel.trim().to_owned()
+ }
+
+ /// Try to find the relative path of `bindir`, otherwise return it in full.
+ pub fn bindir_relative(&self) -> &Path {
+ let bindir = &self.bindir;
+ if bindir.is_absolute() {
+ // Try to make it relative to the prefix.
+ if let Some(prefix) = &self.prefix {
+ if let Ok(stripped) = bindir.strip_prefix(prefix) {
+ return stripped;
+ }
+ }
+ }
+ bindir
+ }
+
+ /// Try to find the relative path of `libdir`.
+ pub fn libdir_relative(&self) -> Option<&Path> {
+ let libdir = self.libdir.as_ref()?;
+ if libdir.is_relative() {
+ Some(libdir)
+ } else {
+ // Try to make it relative to the prefix.
+ libdir.strip_prefix(self.prefix.as_ref()?).ok()
+ }
+ }
+
+ /// The absolute path to the downloaded LLVM artifacts.
+ pub(crate) fn ci_llvm_root(&self) -> PathBuf {
+ assert!(self.llvm_from_ci);
+ self.out.join(&*self.build.triple).join("ci-llvm")
+ }
+
+ /// Determine whether llvm should be linked dynamically.
+ ///
+ /// If `false`, llvm should be linked statically.
+ /// This is computed on demand since LLVM might have to first be downloaded from CI.
+ pub(crate) fn llvm_link_shared(builder: &Builder<'_>) -> bool {
+ let mut opt = builder.config.llvm_link_shared.get();
+ if opt.is_none() && builder.config.dry_run {
+ // just assume static for now - dynamic linking isn't supported on all platforms
+ return false;
+ }
+
+ let llvm_link_shared = *opt.get_or_insert_with(|| {
+ if builder.config.llvm_from_ci {
+ crate::native::maybe_download_ci_llvm(builder);
+ let ci_llvm = builder.config.ci_llvm_root();
+ let link_type = t!(
+ std::fs::read_to_string(ci_llvm.join("link-type.txt")),
+ format!("CI llvm missing: {}", ci_llvm.display())
+ );
+ link_type == "dynamic"
+ } else {
+ // unclear how thought-through this default is, but it maintains compatibility with
+ // previous behavior
+ false
+ }
+ });
+ builder.config.llvm_link_shared.set(opt);
+ llvm_link_shared
+ }
+
+ /// Return whether we will use a downloaded, pre-compiled version of rustc, or just build from source.
+ pub(crate) fn download_rustc(builder: &Builder<'_>) -> bool {
+ static DOWNLOAD_RUSTC: OnceCell<bool> = OnceCell::new();
+ if builder.config.dry_run && DOWNLOAD_RUSTC.get().is_none() {
+ // avoid trying to actually download the commit
+ return false;
+ }
+
+ *DOWNLOAD_RUSTC.get_or_init(|| match &builder.config.download_rustc_commit {
+ None => false,
+ Some(commit) => {
+ download_ci_rustc(builder, commit);
+ true
+ }
+ })
+ }
+
+ pub(crate) fn initial_rustfmt(builder: &Builder<'_>) -> Option<PathBuf> {
+ match &mut *builder.config.initial_rustfmt.borrow_mut() {
+ RustfmtState::SystemToolchain(p) | RustfmtState::Downloaded(p) => Some(p.clone()),
+ RustfmtState::Unavailable => None,
+ r @ RustfmtState::LazyEvaluated => {
+ if builder.config.dry_run {
+ return Some(PathBuf::new());
+ }
+ let path = maybe_download_rustfmt(builder);
+ *r = if let Some(p) = &path {
+ RustfmtState::Downloaded(p.clone())
+ } else {
+ RustfmtState::Unavailable
+ };
+ path
+ }
+ }
+ }
+
+ pub fn verbose(&self) -> bool {
+ self.verbose > 0
+ }
+
+ pub fn sanitizers_enabled(&self, target: TargetSelection) -> bool {
+ self.target_config.get(&target).map(|t| t.sanitizers).flatten().unwrap_or(self.sanitizers)
+ }
+
+ pub fn any_sanitizers_enabled(&self) -> bool {
+ self.target_config.values().any(|t| t.sanitizers == Some(true)) || self.sanitizers
+ }
+
+ pub fn profiler_enabled(&self, target: TargetSelection) -> bool {
+ self.target_config.get(&target).map(|t| t.profiler).flatten().unwrap_or(self.profiler)
+ }
+
+ pub fn any_profiler_enabled(&self) -> bool {
+ self.target_config.values().any(|t| t.profiler == Some(true)) || self.profiler
+ }
+
+ pub fn llvm_enabled(&self) -> bool {
+ self.rust_codegen_backends.contains(&INTERNER.intern_str("llvm"))
+ }
+
+ pub fn llvm_libunwind(&self, target: TargetSelection) -> LlvmLibunwind {
+ self.target_config
+ .get(&target)
+ .and_then(|t| t.llvm_libunwind)
+ .or(self.llvm_libunwind_default)
+ .unwrap_or(LlvmLibunwind::No)
+ }
+
+ pub fn submodules(&self, rust_info: &GitInfo) -> bool {
+ self.submodules.unwrap_or(rust_info.is_git())
+ }
+}
+
+fn set<T>(field: &mut T, val: Option<T>) {
+ if let Some(v) = val {
+ *field = v;
+ }
+}
+
+fn threads_from_config(v: u32) -> u32 {
+ match v {
+ 0 => num_cpus::get() as u32,
+ n => n,
+ }
+}
+
+/// Returns the commit to download, or `None` if we shouldn't download CI artifacts.
+fn download_ci_rustc_commit(
+ config: &Config,
+ download_rustc: Option<StringOrBool>,
+) -> Option<String> {
+ // If `download-rustc` is not set, default to rebuilding.
+ let if_unchanged = match download_rustc {
+ None | Some(StringOrBool::Bool(false)) => return None,
+ Some(StringOrBool::Bool(true)) => false,
+ Some(StringOrBool::String(s)) if s == "if-unchanged" => true,
+ Some(StringOrBool::String(other)) => {
+ panic!("unrecognized option for download-rustc: {}", other)
+ }
+ };
+
+ // Handle running from a directory other than the top level
+ let top_level = output(config.git().args(&["rev-parse", "--show-toplevel"]));
+ let top_level = top_level.trim_end();
+ let compiler = format!("{top_level}/compiler/");
+ let library = format!("{top_level}/library/");
+
+ // Look for a version to compare to based on the current commit.
+ // Only commits merged by bors will have CI artifacts.
+ let merge_base = output(
+ config
+ .git()
+ .arg("rev-list")
+ .arg(format!("--author={}", config.stage0_metadata.config.git_merge_commit_email))
+ .args(&["-n1", "--first-parent", "HEAD"]),
+ );
+ let commit = merge_base.trim_end();
+ if commit.is_empty() {
+ println!("error: could not find commit hash for downloading rustc");
+ println!("help: maybe your repository history is too shallow?");
+ println!("help: consider disabling `download-rustc`");
+ println!("help: or fetch enough history to include one upstream commit");
+ crate::detail_exit(1);
+ }
+
+ // Warn if there were changes to the compiler or standard library since the ancestor commit.
+ let has_changes = !t!(config
+ .git()
+ .args(&["diff-index", "--quiet", &commit, "--", &compiler, &library])
+ .status())
+ .success();
+ if has_changes {
+ if if_unchanged {
+ if config.verbose > 0 {
+ println!(
+ "warning: saw changes to compiler/ or library/ since {commit}; \
+ ignoring `download-rustc`"
+ );
+ }
+ return None;
+ }
+ println!(
+ "warning: `download-rustc` is enabled, but there are changes to \
+ compiler/ or library/"
+ );
+ }
+
+ Some(commit.to_string())
+}
+
+fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option<PathBuf> {
+ let RustfmtMetadata { date, version } = builder.config.stage0_metadata.rustfmt.as_ref()?;
+ let channel = format!("{version}-{date}");
+
+ let host = builder.config.build;
+ let rustfmt_path = builder.config.initial_rustc.with_file_name(exe("rustfmt", host));
+ let bin_root = builder.config.out.join(host.triple).join("stage0");
+ let rustfmt_stamp = bin_root.join(".rustfmt-stamp");
+ if rustfmt_path.exists() && !program_out_of_date(&rustfmt_stamp, &channel) {
+ return Some(rustfmt_path);
+ }
+
+ let filename = format!("rustfmt-{version}-{build}.tar.xz", build = host.triple);
+ download_component(builder, DownloadSource::Dist, filename, "rustfmt-preview", &date, "stage0");
+
+ builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt"));
+ builder.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt"));
+
+ builder.create(&rustfmt_stamp, &channel);
+ Some(rustfmt_path)
+}
+
+fn download_ci_rustc(builder: &Builder<'_>, commit: &str) {
+ builder.verbose(&format!("using downloaded stage2 artifacts from CI (commit {commit})"));
+ let channel = builder.config.artifact_channel(commit);
+ let host = builder.config.build.triple;
+ let bin_root = builder.out.join(host).join("ci-rustc");
+ let rustc_stamp = bin_root.join(".rustc-stamp");
+
+ if !bin_root.join("bin").join("rustc").exists() || program_out_of_date(&rustc_stamp, commit) {
+ if bin_root.exists() {
+ t!(fs::remove_dir_all(&bin_root));
+ }
+ let filename = format!("rust-std-{channel}-{host}.tar.xz");
+ let pattern = format!("rust-std-{host}");
+ download_ci_component(builder, filename, &pattern, commit);
+ let filename = format!("rustc-{channel}-{host}.tar.xz");
+ download_ci_component(builder, filename, "rustc", commit);
+ // download-rustc doesn't need its own cargo, it can just use beta's.
+ let filename = format!("rustc-dev-{channel}-{host}.tar.xz");
+ download_ci_component(builder, filename, "rustc-dev", commit);
+
+ builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustc"));
+ builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc"));
+ let lib_dir = bin_root.join("lib");
+ for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) {
+ let lib = t!(lib);
+ if lib.path().extension() == Some(OsStr::new("so")) {
+ builder.fix_bin_or_dylib(&lib.path());
+ }
+ }
+ t!(fs::write(rustc_stamp, commit));
+ }
+}
+
+pub(crate) enum DownloadSource {
+ CI,
+ Dist,
+}
+
+/// Download a single component of a CI-built toolchain (not necessarily a published nightly).
+// NOTE: intentionally takes an owned string to avoid downloading multiple times by accident
+fn download_ci_component(builder: &Builder<'_>, filename: String, prefix: &str, commit: &str) {
+ download_component(builder, DownloadSource::CI, filename, prefix, commit, "ci-rustc")
+}
+
+fn download_component(
+ builder: &Builder<'_>,
+ mode: DownloadSource,
+ filename: String,
+ prefix: &str,
+ key: &str,
+ destination: &str,
+) {
+ let cache_dst = builder.out.join("cache");
+ let cache_dir = cache_dst.join(key);
+ if !cache_dir.exists() {
+ t!(fs::create_dir_all(&cache_dir));
+ }
+
+ let bin_root = builder.out.join(builder.config.build.triple).join(destination);
+ let tarball = cache_dir.join(&filename);
+ let (base_url, url, should_verify) = match mode {
+ DownloadSource::CI => (
+ builder.config.stage0_metadata.config.artifacts_server.clone(),
+ format!("{key}/{filename}"),
+ false,
+ ),
+ DownloadSource::Dist => {
+ let dist_server = env::var("RUSTUP_DIST_SERVER")
+ .unwrap_or(builder.config.stage0_metadata.config.dist_server.to_string());
+ // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json
+ (dist_server, format!("dist/{key}/{filename}"), true)
+ }
+ };
+
+ // For the beta compiler, put special effort into ensuring the checksums are valid.
+ // FIXME: maybe we should do this for download-rustc as well? but it would be a pain to update
+ // this on each and every nightly ...
+ let checksum = if should_verify {
+ let error = format!(
+ "src/stage0.json doesn't contain a checksum for {url}. \
+ Pre-built artifacts might not be available for this \
+ target at this time, see https://doc.rust-lang.org/nightly\
+ /rustc/platform-support.html for more information."
+ );
+ let sha256 = builder.config.stage0_metadata.checksums_sha256.get(&url).expect(&error);
+ if tarball.exists() {
+ if builder.verify(&tarball, sha256) {
+ builder.unpack(&tarball, &bin_root, prefix);
+ return;
+ } else {
+ builder.verbose(&format!(
+ "ignoring cached file {} due to failed verification",
+ tarball.display()
+ ));
+ builder.remove(&tarball);
+ }
+ }
+ Some(sha256)
+ } else if tarball.exists() {
+ builder.unpack(&tarball, &bin_root, prefix);
+ return;
+ } else {
+ None
+ };
+
+ builder.download_component(&format!("{base_url}/{url}"), &tarball, "");
+ if let Some(sha256) = checksum {
+ if !builder.verify(&tarball, sha256) {
+ panic!("failed to verify {}", tarball.display());
+ }
+ }
+
+ builder.unpack(&tarball, &bin_root, prefix);
+}
diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py
new file mode 100755
index 000000000..6b139decb
--- /dev/null
+++ b/src/bootstrap/configure.py
@@ -0,0 +1,493 @@
+#!/usr/bin/env python
+
+# ignore-tidy-linelength
+
+from __future__ import absolute_import, division, print_function
+import sys
+import os
+rust_dir = os.path.dirname(os.path.abspath(__file__))
+rust_dir = os.path.dirname(rust_dir)
+rust_dir = os.path.dirname(rust_dir)
+sys.path.append(os.path.join(rust_dir, "src", "bootstrap"))
+import bootstrap
+
+
+class Option(object):
+ def __init__(self, name, rustbuild, desc, value):
+ self.name = name
+ self.rustbuild = rustbuild
+ self.desc = desc
+ self.value = value
+
+
+options = []
+
+
+def o(*args):
+ options.append(Option(*args, value=False))
+
+
+def v(*args):
+ options.append(Option(*args, value=True))
+
+
+o("debug", "rust.debug", "enables debugging environment; does not affect optimization of bootstrapped code")
+o("docs", "build.docs", "build standard library documentation")
+o("compiler-docs", "build.compiler-docs", "build compiler documentation")
+o("optimize-tests", "rust.optimize-tests", "build tests with optimizations")
+o("verbose-tests", "rust.verbose-tests", "enable verbose output when running tests")
+o("ccache", "llvm.ccache", "invoke gcc/clang via ccache to reuse object files between builds")
+o("sccache", None, "invoke gcc/clang via sccache to reuse object files between builds")
+o("local-rust", None, "use an installed rustc rather than downloading a snapshot")
+v("local-rust-root", None, "set prefix for local rust binary")
+o("local-rebuild", "build.local-rebuild", "assume local-rust matches the current version, for rebuilds; implies local-rust, and is implied if local-rust already matches the current version")
+o("llvm-static-stdcpp", "llvm.static-libstdcpp", "statically link to libstdc++ for LLVM")
+o("llvm-link-shared", "llvm.link-shared", "prefer shared linking to LLVM (llvm-config --link-shared)")
+o("rpath", "rust.rpath", "build rpaths into rustc itself")
+o("llvm-version-check", "llvm.version-check", "check if the LLVM version is supported, build anyway")
+o("codegen-tests", "rust.codegen-tests", "run the src/test/codegen tests")
+o("option-checking", None, "complain about unrecognized options in this configure script")
+o("ninja", "llvm.ninja", "build LLVM using the Ninja generator (for MSVC, requires building in the correct environment)")
+o("locked-deps", "build.locked-deps", "force Cargo.lock to be up to date")
+o("vendor", "build.vendor", "enable usage of vendored Rust crates")
+o("sanitizers", "build.sanitizers", "build the sanitizer runtimes (asan, lsan, msan, tsan, hwasan)")
+o("dist-src", "rust.dist-src", "when building tarballs enables building a source tarball")
+o("cargo-native-static", "build.cargo-native-static", "static native libraries in cargo")
+o("profiler", "build.profiler", "build the profiler runtime")
+o("full-tools", None, "enable all tools")
+o("lld", "rust.lld", "build lld")
+o("clang", "llvm.clang", "build clang")
+o("missing-tools", "dist.missing-tools", "allow failures when building tools")
+o("use-libcxx", "llvm.use-libcxx", "build LLVM with libc++")
+o("control-flow-guard", "rust.control-flow-guard", "Enable Control Flow Guard")
+
+v("llvm-cflags", "llvm.cflags", "build LLVM with these extra compiler flags")
+v("llvm-cxxflags", "llvm.cxxflags", "build LLVM with these extra compiler flags")
+v("llvm-ldflags", "llvm.ldflags", "build LLVM with these extra linker flags")
+
+v("llvm-libunwind", "rust.llvm-libunwind", "use LLVM libunwind")
+
+# Optimization and debugging options. These may be overridden by the release
+# channel, etc.
+o("optimize-llvm", "llvm.optimize", "build optimized LLVM")
+o("llvm-assertions", "llvm.assertions", "build LLVM with assertions")
+o("llvm-plugins", "llvm.plugins", "build LLVM with plugin interface")
+o("debug-assertions", "rust.debug-assertions", "build with debugging assertions")
+o("debug-assertions-std", "rust.debug-assertions-std", "build the standard library with debugging assertions")
+o("overflow-checks", "rust.overflow-checks", "build with overflow checks")
+o("overflow-checks-std", "rust.overflow-checks-std", "build the standard library with overflow checks")
+o("llvm-release-debuginfo", "llvm.release-debuginfo", "build LLVM with debugger metadata")
+v("debuginfo-level", "rust.debuginfo-level", "debuginfo level for Rust code")
+v("debuginfo-level-rustc", "rust.debuginfo-level-rustc", "debuginfo level for the compiler")
+v("debuginfo-level-std", "rust.debuginfo-level-std", "debuginfo level for the standard library")
+v("debuginfo-level-tools", "rust.debuginfo-level-tools", "debuginfo level for the tools")
+v("debuginfo-level-tests", "rust.debuginfo-level-tests", "debuginfo level for the test suites run with compiletest")
+v("save-toolstates", "rust.save-toolstates", "save build and test status of external tools into this file")
+
+v("prefix", "install.prefix", "set installation prefix")
+v("localstatedir", "install.localstatedir", "local state directory")
+v("datadir", "install.datadir", "install data")
+v("sysconfdir", "install.sysconfdir", "install system configuration files")
+v("infodir", "install.infodir", "install additional info")
+v("libdir", "install.libdir", "install libraries")
+v("mandir", "install.mandir", "install man pages in PATH")
+v("docdir", "install.docdir", "install documentation in PATH")
+v("bindir", "install.bindir", "install binaries")
+
+v("llvm-root", None, "set LLVM root")
+v("llvm-config", None, "set path to llvm-config")
+v("llvm-filecheck", None, "set path to LLVM's FileCheck utility")
+v("python", "build.python", "set path to python")
+v("android-cross-path", "target.arm-linux-androideabi.android-ndk",
+ "Android NDK standalone path (deprecated)")
+v("i686-linux-android-ndk", "target.i686-linux-android.android-ndk",
+ "i686-linux-android NDK standalone path")
+v("arm-linux-androideabi-ndk", "target.arm-linux-androideabi.android-ndk",
+ "arm-linux-androideabi NDK standalone path")
+v("armv7-linux-androideabi-ndk", "target.armv7-linux-androideabi.android-ndk",
+ "armv7-linux-androideabi NDK standalone path")
+v("thumbv7neon-linux-androideabi-ndk", "target.thumbv7neon-linux-androideabi.android-ndk",
+ "thumbv7neon-linux-androideabi NDK standalone path")
+v("aarch64-linux-android-ndk", "target.aarch64-linux-android.android-ndk",
+ "aarch64-linux-android NDK standalone path")
+v("x86_64-linux-android-ndk", "target.x86_64-linux-android.android-ndk",
+ "x86_64-linux-android NDK standalone path")
+v("musl-root", "target.x86_64-unknown-linux-musl.musl-root",
+ "MUSL root installation directory (deprecated)")
+v("musl-root-x86_64", "target.x86_64-unknown-linux-musl.musl-root",
+ "x86_64-unknown-linux-musl install directory")
+v("musl-root-i586", "target.i586-unknown-linux-musl.musl-root",
+ "i586-unknown-linux-musl install directory")
+v("musl-root-i686", "target.i686-unknown-linux-musl.musl-root",
+ "i686-unknown-linux-musl install directory")
+v("musl-root-arm", "target.arm-unknown-linux-musleabi.musl-root",
+ "arm-unknown-linux-musleabi install directory")
+v("musl-root-armhf", "target.arm-unknown-linux-musleabihf.musl-root",
+ "arm-unknown-linux-musleabihf install directory")
+v("musl-root-armv5te", "target.armv5te-unknown-linux-musleabi.musl-root",
+ "armv5te-unknown-linux-musleabi install directory")
+v("musl-root-armv7", "target.armv7-unknown-linux-musleabi.musl-root",
+ "armv7-unknown-linux-musleabi install directory")
+v("musl-root-armv7hf", "target.armv7-unknown-linux-musleabihf.musl-root",
+ "armv7-unknown-linux-musleabihf install directory")
+v("musl-root-aarch64", "target.aarch64-unknown-linux-musl.musl-root",
+ "aarch64-unknown-linux-musl install directory")
+v("musl-root-mips", "target.mips-unknown-linux-musl.musl-root",
+ "mips-unknown-linux-musl install directory")
+v("musl-root-mipsel", "target.mipsel-unknown-linux-musl.musl-root",
+ "mipsel-unknown-linux-musl install directory")
+v("musl-root-mips64", "target.mips64-unknown-linux-muslabi64.musl-root",
+ "mips64-unknown-linux-muslabi64 install directory")
+v("musl-root-mips64el", "target.mips64el-unknown-linux-muslabi64.musl-root",
+ "mips64el-unknown-linux-muslabi64 install directory")
+v("qemu-armhf-rootfs", "target.arm-unknown-linux-gnueabihf.qemu-rootfs",
+ "rootfs in qemu testing, you probably don't want to use this")
+v("qemu-aarch64-rootfs", "target.aarch64-unknown-linux-gnu.qemu-rootfs",
+ "rootfs in qemu testing, you probably don't want to use this")
+v("qemu-riscv64-rootfs", "target.riscv64gc-unknown-linux-gnu.qemu-rootfs",
+ "rootfs in qemu testing, you probably don't want to use this")
+v("experimental-targets", "llvm.experimental-targets",
+ "experimental LLVM targets to build")
+v("release-channel", "rust.channel", "the name of the release channel to build")
+v("release-description", "rust.description", "optional descriptive string for version output")
+v("dist-compression-formats", None,
+ "comma-separated list of compression formats to use")
+
+# Used on systems where "cc" is unavailable
+v("default-linker", "rust.default-linker", "the default linker")
+
+# Many of these are saved below during the "writing configuration" step
+# (others are conditionally saved).
+o("manage-submodules", "build.submodules", "let the build manage the git submodules")
+o("full-bootstrap", "build.full-bootstrap", "build three compilers instead of two (not recommended except for testing reproducible builds)")
+o("extended", "build.extended", "build an extended rust tool set")
+
+v("tools", None, "List of extended tools will be installed")
+v("codegen-backends", None, "List of codegen backends to build")
+v("build", "build.build", "GNUs ./configure syntax LLVM build triple")
+v("host", None, "GNUs ./configure syntax LLVM host triples")
+v("target", None, "GNUs ./configure syntax LLVM target triples")
+
+v("set", None, "set arbitrary key/value pairs in TOML configuration")
+
+
+def p(msg):
+ print("configure: " + msg)
+
+
+def err(msg):
+ print("configure: error: " + msg)
+ sys.exit(1)
+
+
+if '--help' in sys.argv or '-h' in sys.argv:
+ print('Usage: ./configure [options]')
+ print('')
+ print('Options')
+ for option in options:
+ if 'android' in option.name:
+ # no one needs to know about these obscure options
+ continue
+ if option.value:
+ print('\t{:30} {}'.format('--{}=VAL'.format(option.name), option.desc))
+ else:
+ print('\t{:30} {}'.format('--enable-{}'.format(option.name), option.desc))
+ print('')
+ print('This configure script is a thin configuration shim over the true')
+ print('configuration system, `config.toml`. You can explore the comments')
+ print('in `config.toml.example` next to this configure script to see')
+ print('more information about what each option is. Additionally you can')
+ print('pass `--set` as an argument to set arbitrary key/value pairs')
+ print('in the TOML configuration if desired')
+ print('')
+ print('Also note that all options which take `--enable` can similarly')
+ print('be passed with `--disable-foo` to forcibly disable the option')
+ sys.exit(0)
+
+# Parse all command line arguments into one of these three lists, handling
+# boolean and value-based options separately
+unknown_args = []
+need_value_args = []
+known_args = {}
+
+p("processing command line")
+i = 1
+while i < len(sys.argv):
+ arg = sys.argv[i]
+ i += 1
+ if not arg.startswith('--'):
+ unknown_args.append(arg)
+ continue
+
+ found = False
+ for option in options:
+ value = None
+ if option.value:
+ keyval = arg[2:].split('=', 1)
+ key = keyval[0]
+ if option.name != key:
+ continue
+
+ if len(keyval) > 1:
+ value = keyval[1]
+ elif i < len(sys.argv):
+ value = sys.argv[i]
+ i += 1
+ else:
+ need_value_args.append(arg)
+ continue
+ else:
+ if arg[2:] == 'enable-' + option.name:
+ value = True
+ elif arg[2:] == 'disable-' + option.name:
+ value = False
+ else:
+ continue
+
+ found = True
+ if option.name not in known_args:
+ known_args[option.name] = []
+ known_args[option.name].append((option, value))
+ break
+
+ if not found:
+ unknown_args.append(arg)
+p("")
+
+# Note: here and a few other places, we use [-1] to apply the *last* value
+# passed. But if option-checking is enabled, then the known_args loop will
+# also assert that options are only passed once.
+option_checking = ('option-checking' not in known_args
+ or known_args['option-checking'][-1][1])
+if option_checking:
+ if len(unknown_args) > 0:
+ err("Option '" + unknown_args[0] + "' is not recognized")
+ if len(need_value_args) > 0:
+ err("Option '{0}' needs a value ({0}=val)".format(need_value_args[0]))
+
+# Parse all known arguments into a configuration structure that reflects the
+# TOML we're going to write out
+config = {}
+
+
+def build():
+ if 'build' in known_args:
+ return known_args['build'][-1][1]
+ return bootstrap.default_build_triple(verbose=False)
+
+
+def set(key, value):
+ if isinstance(value, list):
+ # Remove empty values, which value.split(',') tends to generate.
+ value = [v for v in value if v]
+
+ s = "{:20} := {}".format(key, value)
+ if len(s) < 70:
+ p(s)
+ else:
+ p(s[:70] + " ...")
+
+ arr = config
+ parts = key.split('.')
+ for i, part in enumerate(parts):
+ if i == len(parts) - 1:
+ arr[part] = value
+ else:
+ if part not in arr:
+ arr[part] = {}
+ arr = arr[part]
+
+
+for key in known_args:
+ # The `set` option is special and can be passed a bunch of times
+ if key == 'set':
+ for option, value in known_args[key]:
+ keyval = value.split('=', 1)
+ if len(keyval) == 1 or keyval[1] == "true":
+ value = True
+ elif keyval[1] == "false":
+ value = False
+ else:
+ value = keyval[1]
+ set(keyval[0], value)
+ continue
+
+ # Ensure each option is only passed once
+ arr = known_args[key]
+ if option_checking and len(arr) > 1:
+ err("Option '{}' provided more than once".format(key))
+ option, value = arr[-1]
+
+ # If we have a clear avenue to set our value in rustbuild, do so
+ if option.rustbuild is not None:
+ set(option.rustbuild, value)
+ continue
+
+ # Otherwise we're a "special" option and need some extra handling, so do
+ # that here.
+ if option.name == 'sccache':
+ set('llvm.ccache', 'sccache')
+ elif option.name == 'local-rust':
+ for path in os.environ['PATH'].split(os.pathsep):
+ if os.path.exists(path + '/rustc'):
+ set('build.rustc', path + '/rustc')
+ break
+ for path in os.environ['PATH'].split(os.pathsep):
+ if os.path.exists(path + '/cargo'):
+ set('build.cargo', path + '/cargo')
+ break
+ elif option.name == 'local-rust-root':
+ set('build.rustc', value + '/bin/rustc')
+ set('build.cargo', value + '/bin/cargo')
+ elif option.name == 'llvm-root':
+ set('target.{}.llvm-config'.format(build()), value + '/bin/llvm-config')
+ elif option.name == 'llvm-config':
+ set('target.{}.llvm-config'.format(build()), value)
+ elif option.name == 'llvm-filecheck':
+ set('target.{}.llvm-filecheck'.format(build()), value)
+ elif option.name == 'tools':
+ set('build.tools', value.split(','))
+ elif option.name == 'codegen-backends':
+ set('rust.codegen-backends', value.split(','))
+ elif option.name == 'host':
+ set('build.host', value.split(','))
+ elif option.name == 'target':
+ set('build.target', value.split(','))
+ elif option.name == 'full-tools':
+ set('rust.codegen-backends', ['llvm'])
+ set('rust.lld', True)
+ set('rust.llvm-tools', True)
+ set('build.extended', True)
+ elif option.name == 'option-checking':
+ # this was handled above
+ pass
+ elif option.name == 'dist-compression-formats':
+ set('dist.compression-formats', value.split(','))
+ else:
+ raise RuntimeError("unhandled option {}".format(option.name))
+
+set('build.configure-args', sys.argv[1:])
+
+# "Parse" the `config.toml.example` file into the various sections, and we'll
+# use this as a template of a `config.toml` to write out which preserves
+# all the various comments and whatnot.
+#
+# Note that the `target` section is handled separately as we'll duplicate it
+# per configured target, so there's a bit of special handling for that here.
+sections = {}
+cur_section = None
+sections[None] = []
+section_order = [None]
+targets = {}
+
+for line in open(rust_dir + '/config.toml.example').read().split("\n"):
+ if line.startswith('['):
+ cur_section = line[1:-1]
+ if cur_section.startswith('target'):
+ cur_section = 'target'
+ elif '.' in cur_section:
+ raise RuntimeError("don't know how to deal with section: {}".format(cur_section))
+ sections[cur_section] = [line]
+ section_order.append(cur_section)
+ else:
+ sections[cur_section].append(line)
+
+# Fill out the `targets` array by giving all configured targets a copy of the
+# `target` section we just loaded from the example config
+configured_targets = [build()]
+if 'build' in config:
+ if 'host' in config['build']:
+ configured_targets += config['build']['host']
+ if 'target' in config['build']:
+ configured_targets += config['build']['target']
+if 'target' in config:
+ for target in config['target']:
+ configured_targets.append(target)
+for target in configured_targets:
+ targets[target] = sections['target'][:]
+ targets[target][0] = targets[target][0].replace("x86_64-unknown-linux-gnu", "'{}'".format(target))
+
+
+def is_number(value):
+ try:
+ float(value)
+ return True
+ except ValueError:
+ return False
+
+
+# Here we walk through the constructed configuration we have from the parsed
+# command line arguments. We then apply each piece of configuration by
+# basically just doing a `sed` to change the various configuration line to what
+# we've got configure.
+def to_toml(value):
+ if isinstance(value, bool):
+ if value:
+ return "true"
+ else:
+ return "false"
+ elif isinstance(value, list):
+ return '[' + ', '.join(map(to_toml, value)) + ']'
+ elif isinstance(value, str):
+ # Don't put quotes around numeric values
+ if is_number(value):
+ return value
+ else:
+ return "'" + value + "'"
+ else:
+ raise RuntimeError('no toml')
+
+
+def configure_section(lines, config):
+ for key in config:
+ value = config[key]
+ found = False
+ for i, line in enumerate(lines):
+ if not line.startswith('#' + key + ' = '):
+ continue
+ found = True
+ lines[i] = "{} = {}".format(key, to_toml(value))
+ break
+ if not found:
+ # These are used by rpm, but aren't accepted by x.py.
+ # Give a warning that they're ignored, but not a hard error.
+ if key in ["infodir", "localstatedir"]:
+ print("warning: {} will be ignored".format(key))
+ else:
+ raise RuntimeError("failed to find config line for {}".format(key))
+
+
+for section_key in config:
+ section_config = config[section_key]
+ if section_key not in sections:
+ raise RuntimeError("config key {} not in sections".format(section_key))
+
+ if section_key == 'target':
+ for target in section_config:
+ configure_section(targets[target], section_config[target])
+ else:
+ configure_section(sections[section_key], section_config)
+
+# Now that we've built up our `config.toml`, write it all out in the same
+# order that we read it in.
+p("")
+p("writing `config.toml` in current directory")
+with bootstrap.output('config.toml') as f:
+ for section in section_order:
+ if section == 'target':
+ for target in targets:
+ for line in targets[target]:
+ f.write(line + "\n")
+ else:
+ for line in sections[section]:
+ f.write(line + "\n")
+
+with bootstrap.output('Makefile') as f:
+ contents = os.path.join(rust_dir, 'src', 'bootstrap', 'mk', 'Makefile.in')
+ contents = open(contents).read()
+ contents = contents.replace("$(CFG_SRC_DIR)", rust_dir + '/')
+ contents = contents.replace("$(CFG_PYTHON)", sys.executable)
+ f.write(contents)
+
+p("")
+p("run `python {}/x.py --help`".format(rust_dir))
diff --git a/src/bootstrap/defaults/README.md b/src/bootstrap/defaults/README.md
new file mode 100644
index 000000000..f5b96db1b
--- /dev/null
+++ b/src/bootstrap/defaults/README.md
@@ -0,0 +1,12 @@
+# About bootstrap defaults
+
+These defaults are intended to be a good starting point for working with x.py,
+with the understanding that no one set of defaults make sense for everyone.
+
+They are still experimental, and we'd appreciate your help improving them!
+If you use a setting that's not in these defaults that you think
+others would benefit from, please [file an issue] or make a PR with the changes.
+Similarly, if one of these defaults doesn't match what you use personally,
+please open an issue to get it changed.
+
+[file an issue]: https://github.com/rust-lang/rust/issues/new/choose
diff --git a/src/bootstrap/defaults/config.codegen.toml b/src/bootstrap/defaults/config.codegen.toml
new file mode 100644
index 000000000..088cbd105
--- /dev/null
+++ b/src/bootstrap/defaults/config.codegen.toml
@@ -0,0 +1,19 @@
+# These defaults are meant for contributors to the compiler who modify codegen or LLVM
+[build]
+# Contributors working on the compiler will probably expect compiler docs to be generated.
+compiler-docs = true
+
+[llvm]
+# This enables debug-assertions in LLVM,
+# catching logic errors in codegen much earlier in the process.
+assertions = true
+
+[rust]
+# This enables `RUSTC_LOG=debug`, avoiding confusing situations
+# where adding `debug!()` appears to do nothing.
+# However, it makes running the compiler slightly slower.
+debug-logging = true
+# This greatly increases the speed of rebuilds, especially when there are only minor changes. However, it makes the initial build slightly slower.
+incremental = true
+# Print backtrace on internal compiler errors during bootstrap
+backtrace-on-ice = true
diff --git a/src/bootstrap/defaults/config.compiler.toml b/src/bootstrap/defaults/config.compiler.toml
new file mode 100644
index 000000000..2f4ccb825
--- /dev/null
+++ b/src/bootstrap/defaults/config.compiler.toml
@@ -0,0 +1,18 @@
+# These defaults are meant for contributors to the compiler who do not modify codegen or LLVM
+[build]
+# Contributors working on the compiler will probably expect compiler docs to be generated.
+compiler-docs = true
+
+[rust]
+# This enables `RUSTC_LOG=debug`, avoiding confusing situations
+# where adding `debug!()` appears to do nothing.
+# However, it makes running the compiler slightly slower.
+debug-logging = true
+# This greatly increases the speed of rebuilds, especially when there are only minor changes. However, it makes the initial build slightly slower.
+incremental = true
+# Print backtrace on internal compiler errors during bootstrap
+backtrace-on-ice = true
+
+[llvm]
+# Will download LLVM from CI if available on your platform.
+download-ci-llvm = "if-available"
diff --git a/src/bootstrap/defaults/config.library.toml b/src/bootstrap/defaults/config.library.toml
new file mode 100644
index 000000000..7bc054d3a
--- /dev/null
+++ b/src/bootstrap/defaults/config.library.toml
@@ -0,0 +1,14 @@
+# These defaults are meant for contributors to the standard library and documentation.
+[build]
+# When building the standard library, you almost never want to build the compiler itself.
+build-stage = 0
+test-stage = 0
+bench-stage = 0
+
+[rust]
+# This greatly increases the speed of rebuilds, especially when there are only minor changes. However, it makes the initial build slightly slower.
+incremental = true
+
+[llvm]
+# Will download LLVM from CI if available on your platform.
+download-ci-llvm = "if-available"
diff --git a/src/bootstrap/defaults/config.tools.toml b/src/bootstrap/defaults/config.tools.toml
new file mode 100644
index 000000000..6b6625342
--- /dev/null
+++ b/src/bootstrap/defaults/config.tools.toml
@@ -0,0 +1,22 @@
+# These defaults are meant for contributors to tools which build on the
+# compiler, but do not modify it directly.
+[rust]
+# This enables `RUSTC_LOG=debug`, avoiding confusing situations
+# where adding `debug!()` appears to do nothing.
+# However, it makes running the compiler slightly slower.
+debug-logging = true
+# This greatly increases the speed of rebuilds, especially when there are only minor changes. However, it makes the initial build slightly slower.
+incremental = true
+# Download rustc from CI instead of building it from source.
+# This cuts compile times by almost 60x, but means you can't modify the compiler.
+download-rustc = "if-unchanged"
+
+[build]
+# Document with the in-tree rustdoc by default, since `download-rustc` makes it quick to compile.
+doc-stage = 2
+# Contributors working on tools will probably expect compiler docs to be generated, so they can figure out how to use the API.
+compiler-docs = true
+
+[llvm]
+# Will download LLVM from CI if available on your platform.
+download-ci-llvm = "if-available"
diff --git a/src/bootstrap/defaults/config.user.toml b/src/bootstrap/defaults/config.user.toml
new file mode 100644
index 000000000..6647061d8
--- /dev/null
+++ b/src/bootstrap/defaults/config.user.toml
@@ -0,0 +1,9 @@
+# These defaults are meant for users and distro maintainers building from source, without intending to make multiple changes.
+[build]
+# When compiling from source, you almost always want a full stage 2 build,
+# which has all the latest optimizations from nightly.
+build-stage = 2
+test-stage = 2
+doc-stage = 2
+# When compiling from source, you usually want all tools.
+extended = true
diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs
new file mode 100644
index 000000000..6291b204e
--- /dev/null
+++ b/src/bootstrap/dist.rs
@@ -0,0 +1,2157 @@
+//! Implementation of the various distribution aspects of the compiler.
+//!
+//! This module is responsible for creating tarballs of the standard library,
+//! compiler, and documentation. This ends up being what we distribute to
+//! everyone as well.
+//!
+//! No tarball is actually created literally in this file, but rather we shell
+//! out to `rust-installer` still. This may one day be replaced with bits and
+//! pieces of `rustup.rs`!
+
+use std::collections::HashSet;
+use std::env;
+use std::fs;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
+use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
+use crate::cache::{Interned, INTERNER};
+use crate::compile;
+use crate::config::TargetSelection;
+use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
+use crate::tool::{self, Tool};
+use crate::util::{exe, is_dylib, output, t, timeit};
+use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
+
+pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
+ format!("{}-{}", component, builder.rust_package_vers())
+}
+
+pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
+ builder.out.join("dist")
+}
+
+pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
+ builder.out.join("tmp/dist")
+}
+
+fn missing_tool(tool_name: &str, skip: bool) {
+ if skip {
+ println!("Unable to build {}, skipping dist", tool_name)
+ } else {
+ let help = "note: not all tools are available on all nightlies\nhelp: see https://forge.rust-lang.org/infra/toolstate.html for more information";
+ panic!(
+ "Unable to build submodule tool {} (use `missing-tools = true` to ignore this failure)\n{}",
+ tool_name, help
+ )
+ }
+}
+
+fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
+ if !builder.config.extended {
+ return false;
+ }
+ builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
+}
+
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Docs {
+ pub host: TargetSelection,
+}
+
+impl Step for Docs {
+ type Output = Option<GeneratedTarball>;
+ const DEFAULT: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let default = run.builder.config.docs;
+ run.alias("rust-docs").default_condition(default)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Docs { host: run.target });
+ }
+
+ /// Builds the `rust-docs` installer component.
+ fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
+ let host = self.host;
+ builder.default_doc(&[]);
+
+ let dest = "share/doc/rust/html";
+
+ let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
+ tarball.set_product_name("Rust Documentation");
+ tarball.add_bulk_dir(&builder.doc_out(host), dest);
+ tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
+ Some(tarball.generate())
+ }
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RustcDocs {
+ pub host: TargetSelection,
+}
+
+impl Step for RustcDocs {
+ type Output = Option<GeneratedTarball>;
+ const DEFAULT: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let builder = run.builder;
+ run.alias("rustc-docs").default_condition(builder.config.compiler_docs)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(RustcDocs { host: run.target });
+ }
+
+ /// Builds the `rustc-docs` installer component.
+ fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
+ let host = self.host;
+ builder.default_doc(&[]);
+
+ let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
+ tarball.set_product_name("Rustc Documentation");
+ tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
+ Some(tarball.generate())
+ }
+}
+
+fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
+ let mut found = Vec::with_capacity(files.len());
+
+ for file in files {
+ let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
+
+ if let Some(file_path) = file_path {
+ found.push(file_path);
+ } else {
+ panic!("Could not find '{}' in {:?}", file, path);
+ }
+ }
+
+ found
+}
+
+fn make_win_dist(
+ rust_root: &Path,
+ plat_root: &Path,
+ target: TargetSelection,
+ builder: &Builder<'_>,
+) {
+ //Ask gcc where it keeps its stuff
+ let mut cmd = Command::new(builder.cc(target));
+ cmd.arg("-print-search-dirs");
+ let gcc_out = output(&mut cmd);
+
+ let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
+ let mut lib_path = Vec::new();
+
+ for line in gcc_out.lines() {
+ let idx = line.find(':').unwrap();
+ let key = &line[..idx];
+ let trim_chars: &[_] = &[' ', '='];
+ let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
+
+ if key == "programs" {
+ bin_path.extend(value);
+ } else if key == "libraries" {
+ lib_path.extend(value);
+ }
+ }
+
+ let compiler = if target == "i686-pc-windows-gnu" {
+ "i686-w64-mingw32-gcc.exe"
+ } else if target == "x86_64-pc-windows-gnu" {
+ "x86_64-w64-mingw32-gcc.exe"
+ } else {
+ "gcc.exe"
+ };
+ let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
+ let mut rustc_dlls = vec!["libwinpthread-1.dll"];
+ if target.starts_with("i686-") {
+ rustc_dlls.push("libgcc_s_dw2-1.dll");
+ } else {
+ rustc_dlls.push("libgcc_s_seh-1.dll");
+ }
+
+ let target_libs = [
+ //MinGW libs
+ "libgcc.a",
+ "libgcc_eh.a",
+ "libgcc_s.a",
+ "libm.a",
+ "libmingw32.a",
+ "libmingwex.a",
+ "libstdc++.a",
+ "libiconv.a",
+ "libmoldname.a",
+ "libpthread.a",
+ //Windows import libs
+ "libadvapi32.a",
+ "libbcrypt.a",
+ "libcomctl32.a",
+ "libcomdlg32.a",
+ "libcredui.a",
+ "libcrypt32.a",
+ "libdbghelp.a",
+ "libgdi32.a",
+ "libimagehlp.a",
+ "libiphlpapi.a",
+ "libkernel32.a",
+ "libmsimg32.a",
+ "libmsvcrt.a",
+ "libodbc32.a",
+ "libole32.a",
+ "liboleaut32.a",
+ "libopengl32.a",
+ "libpsapi.a",
+ "librpcrt4.a",
+ "libsecur32.a",
+ "libsetupapi.a",
+ "libshell32.a",
+ "libsynchronization.a",
+ "libuser32.a",
+ "libuserenv.a",
+ "libuuid.a",
+ "libwinhttp.a",
+ "libwinmm.a",
+ "libwinspool.a",
+ "libws2_32.a",
+ "libwsock32.a",
+ ];
+
+ //Find mingw artifacts we want to bundle
+ let target_tools = find_files(&target_tools, &bin_path);
+ let rustc_dlls = find_files(&rustc_dlls, &bin_path);
+ let target_libs = find_files(&target_libs, &lib_path);
+
+ // Copy runtime dlls next to rustc.exe
+ let dist_bin_dir = rust_root.join("bin/");
+ fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
+ for src in rustc_dlls {
+ builder.copy_to_folder(&src, &dist_bin_dir);
+ }
+
+ //Copy platform tools to platform-specific bin directory
+ let target_bin_dir = plat_root
+ .join("lib")
+ .join("rustlib")
+ .join(target.triple)
+ .join("bin")
+ .join("self-contained");
+ fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
+ for src in target_tools {
+ builder.copy_to_folder(&src, &target_bin_dir);
+ }
+
+ // Warn windows-gnu users that the bundled GCC cannot compile C files
+ builder.create(
+ &target_bin_dir.join("GCC-WARNING.txt"),
+ "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
+ used as a linker. In order to be able to compile projects containing C code use \
+ the GCC provided by MinGW or Cygwin.",
+ );
+
+ //Copy platform libs to platform-specific lib directory
+ let target_lib_dir = plat_root
+ .join("lib")
+ .join("rustlib")
+ .join(target.triple)
+ .join("lib")
+ .join("self-contained");
+ fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
+ for src in target_libs {
+ builder.copy_to_folder(&src, &target_lib_dir);
+ }
+}
+
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Mingw {
+ pub host: TargetSelection,
+}
+
+impl Step for Mingw {
+ type Output = Option<GeneratedTarball>;
+ const DEFAULT: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.alias("rust-mingw")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Mingw { host: run.target });
+ }
+
+ /// Builds the `rust-mingw` installer component.
+ ///
+ /// This contains all the bits and pieces to run the MinGW Windows targets
+ /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
+ fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
+ let host = self.host;
+ if !host.ends_with("pc-windows-gnu") {
+ return None;
+ }
+
+ let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
+ tarball.set_product_name("Rust MinGW");
+
+ // The first argument is a "temporary directory" which is just
+ // thrown away (this contains the runtime DLLs included in the rustc package
+ // above) and the second argument is where to place all the MinGW components
+ // (which is what we want).
+ make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
+
+ Some(tarball.generate())
+ }
+}
+
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Rustc {
+ pub compiler: Compiler,
+}
+
+impl Step for Rustc {
+ type Output = GeneratedTarball;
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.alias("rustc")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder
+ .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
+ }
+
+ /// Creates the `rustc` installer component.
+ fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
+ let compiler = self.compiler;
+ let host = self.compiler.host;
+
+ let tarball = Tarball::new(builder, "rustc", &host.triple);
+
+ // Prepare the rustc "image", what will actually end up getting installed
+ prepare_image(builder, compiler, tarball.image_dir());
+
+ // On MinGW we've got a few runtime DLL dependencies that we need to
+ // include. The first argument to this script is where to put these DLLs
+ // (the image we're creating), and the second argument is a junk directory
+ // to ignore all other MinGW stuff the script creates.
+ //
+ // On 32-bit MinGW we're always including a DLL which needs some extra
+ // licenses to distribute. On 64-bit MinGW we don't actually distribute
+ // anything requiring us to distribute a license, but it's likely the
+ // install will *also* include the rust-mingw package, which also needs
+ // licenses, so to be safe we just include it here in all MinGW packages.
+ if host.ends_with("pc-windows-gnu") {
+ make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
+ tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
+ }
+
+ return tarball.generate();
+
+ fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
+ let host = compiler.host;
+ let src = builder.sysroot(compiler);
+
+ // Copy rustc/rustdoc binaries
+ t!(fs::create_dir_all(image.join("bin")));
+ builder.cp_r(&src.join("bin"), &image.join("bin"));
+
+ builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
+
+ let ra_proc_macro_srv = builder
+ .ensure(tool::RustAnalyzerProcMacroSrv {
+ compiler: builder.compiler_for(
+ compiler.stage,
+ builder.config.build,
+ compiler.host,
+ ),
+ target: compiler.host,
+ })
+ .expect("rust-analyzer-proc-macro-server always builds");
+ builder.install(&ra_proc_macro_srv, &image.join("libexec"), 0o755);
+
+ let libdir_relative = builder.libdir_relative(compiler);
+
+ // Copy runtime DLLs needed by the compiler
+ if libdir_relative.to_str() != Some("bin") {
+ let libdir = builder.rustc_libdir(compiler);
+ for entry in builder.read_dir(&libdir) {
+ let name = entry.file_name();
+ if let Some(s) = name.to_str() {
+ if is_dylib(s) {
+ // Don't use custom libdir here because ^lib/ will be resolved again
+ // with installer
+ builder.install(&entry.path(), &image.join("lib"), 0o644);
+ }
+ }
+ }
+ }
+
+ // Copy over the codegen backends
+ let backends_src = builder.sysroot_codegen_backends(compiler);
+ let backends_rel = backends_src
+ .strip_prefix(&src)
+ .unwrap()
+ .strip_prefix(builder.sysroot_libdir_relative(compiler))
+ .unwrap();
+ // Don't use custom libdir here because ^lib/ will be resolved again with installer
+ let backends_dst = image.join("lib").join(&backends_rel);
+
+ t!(fs::create_dir_all(&backends_dst));
+ builder.cp_r(&backends_src, &backends_dst);
+
+ // Copy libLLVM.so to the lib dir as well, if needed. While not
+ // technically needed by rustc itself it's needed by lots of other
+ // components like the llvm tools and LLD. LLD is included below and
+ // tools/LLDB come later, so let's just throw it in the rustc
+ // component for now.
+ maybe_install_llvm_runtime(builder, host, image);
+
+ let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
+ t!(fs::create_dir_all(&dst_dir));
+
+ // Copy over lld if it's there
+ if builder.config.lld_enabled {
+ let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
+ let rust_lld = exe("rust-lld", compiler.host);
+ builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
+ // for `-Z gcc-ld=lld`
+ let gcc_lld_src_dir = src_dir.join("gcc-ld");
+ let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
+ t!(fs::create_dir(&gcc_lld_dst_dir));
+ let exe_name = exe("ld", compiler.host);
+ builder.copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
+ }
+
+ // Man pages
+ t!(fs::create_dir_all(image.join("share/man/man1")));
+ let man_src = builder.src.join("src/doc/man");
+ let man_dst = image.join("share/man/man1");
+
+ // don't use our `bootstrap::util::{copy, cp_r}`, because those try
+ // to hardlink, and we don't want to edit the source templates
+ for file_entry in builder.read_dir(&man_src) {
+ let page_src = file_entry.path();
+ let page_dst = man_dst.join(file_entry.file_name());
+ let src_text = t!(std::fs::read_to_string(&page_src));
+ let new_text = src_text.replace("<INSERT VERSION HERE>", &builder.version);
+ t!(std::fs::write(&page_dst, &new_text));
+ t!(fs::copy(&page_src, &page_dst));
+ }
+
+ // Debugger scripts
+ builder
+ .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
+
+ // Misc license info
+ let cp = |file: &str| {
+ builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
+ };
+ cp("COPYRIGHT");
+ cp("LICENSE-APACHE");
+ cp("LICENSE-MIT");
+ cp("README.md");
+ }
+ }
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct DebuggerScripts {
+ pub sysroot: Interned<PathBuf>,
+ pub host: TargetSelection,
+}
+
+impl Step for DebuggerScripts {
+ type Output = ();
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.never()
+ }
+
+ /// Copies debugger scripts for `target` into the `sysroot` specified.
+ fn run(self, builder: &Builder<'_>) {
+ let host = self.host;
+ let sysroot = self.sysroot;
+ let dst = sysroot.join("lib/rustlib/etc");
+ t!(fs::create_dir_all(&dst));
+ let cp_debugger_script = |file: &str| {
+ builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
+ };
+ if host.contains("windows-msvc") {
+ // windbg debugger scripts
+ builder.install(
+ &builder.src.join("src/etc/rust-windbg.cmd"),
+ &sysroot.join("bin"),
+ 0o755,
+ );
+
+ cp_debugger_script("natvis/intrinsic.natvis");
+ cp_debugger_script("natvis/liballoc.natvis");
+ cp_debugger_script("natvis/libcore.natvis");
+ cp_debugger_script("natvis/libstd.natvis");
+ } else {
+ cp_debugger_script("rust_types.py");
+
+ // gdb debugger scripts
+ builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
+ builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
+
+ cp_debugger_script("gdb_load_rust_pretty_printers.py");
+ cp_debugger_script("gdb_lookup.py");
+ cp_debugger_script("gdb_providers.py");
+
+ // lldb debugger scripts
+ builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
+
+ cp_debugger_script("lldb_lookup.py");
+ cp_debugger_script("lldb_providers.py");
+ cp_debugger_script("lldb_commands")
+ }
+ }
+}
+
+fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
+ // The only true set of target libraries came from the build triple, so
+ // let's reduce redundant work by only producing archives from that host.
+ if compiler.host != builder.config.build {
+ builder.info("\tskipping, not a build host");
+ true
+ } else {
+ false
+ }
+}
+
+/// Copy stamped files into an image's `target/lib` directory.
+fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
+ let dst = image.join("lib/rustlib").join(target.triple).join("lib");
+ let self_contained_dst = dst.join("self-contained");
+ t!(fs::create_dir_all(&dst));
+ t!(fs::create_dir_all(&self_contained_dst));
+ for (path, dependency_type) in builder.read_stamp_file(stamp) {
+ if dependency_type == DependencyType::TargetSelfContained {
+ builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
+ } else if dependency_type == DependencyType::Target || builder.config.build == target {
+ builder.copy(&path, &dst.join(path.file_name().unwrap()));
+ }
+ }
+}
+
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Std {
+ pub compiler: Compiler,
+ pub target: TargetSelection,
+}
+
+impl Step for Std {
+ type Output = Option<GeneratedTarball>;
+ const DEFAULT: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.alias("rust-std")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Std {
+ compiler: run.builder.compiler_for(
+ run.builder.top_stage,
+ run.builder.config.build,
+ run.target,
+ ),
+ target: run.target,
+ });
+ }
+
+ fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
+ let compiler = self.compiler;
+ let target = self.target;
+
+ if skip_host_target_lib(builder, compiler) {
+ return None;
+ }
+
+ builder.ensure(compile::Std::new(compiler, target));
+
+ let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
+ tarball.include_target_in_component_name(true);
+
+ let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
+ let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
+ copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
+
+ Some(tarball.generate())
+ }
+}
+
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RustcDev {
+ pub compiler: Compiler,
+ pub target: TargetSelection,
+}
+
+impl Step for RustcDev {
+ type Output = Option<GeneratedTarball>;
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.alias("rustc-dev")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(RustcDev {
+ compiler: run.builder.compiler_for(
+ run.builder.top_stage,
+ run.builder.config.build,
+ run.target,
+ ),
+ target: run.target,
+ });
+ }
+
+ fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
+ let compiler = self.compiler;
+ let target = self.target;
+ if skip_host_target_lib(builder, compiler) {
+ return None;
+ }
+
+ builder.ensure(compile::Rustc::new(compiler, target));
+
+ let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
+
+ let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
+ let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
+ copy_target_libs(builder, target, tarball.image_dir(), &stamp);
+
+ let src_files = &["Cargo.lock"];
+ // This is the reduced set of paths which will become the rustc-dev component
+ // (essentially the compiler crates and all of their path dependencies).
+ copy_src_dirs(
+ builder,
+ &builder.src,
+ &["compiler"],
+ &[],
+ &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
+ );
+ for file in src_files {
+ tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
+ }
+
+ Some(tarball.generate())
+ }
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Analysis {
+ pub compiler: Compiler,
+ pub target: TargetSelection,
+}
+
+impl Step for Analysis {
+ type Output = Option<GeneratedTarball>;
+ const DEFAULT: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let default = should_build_extended_tool(&run.builder, "analysis");
+ run.alias("rust-analysis").default_condition(default)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Analysis {
+ // Find the actual compiler (handling the full bootstrap option) which
+ // produced the save-analysis data because that data isn't copied
+ // through the sysroot uplifting.
+ compiler: run.builder.compiler_for(
+ run.builder.top_stage,
+ run.builder.config.build,
+ run.target,
+ ),
+ target: run.target,
+ });
+ }
+
+ /// Creates a tarball of save-analysis metadata, if available.
+ fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
+ let compiler = self.compiler;
+ let target = self.target;
+ if compiler.host != builder.config.build {
+ return None;
+ }
+
+ builder.ensure(compile::Std::new(compiler, target));
+ let src = builder
+ .stage_out(compiler, Mode::Std)
+ .join(target.triple)
+ .join(builder.cargo_dir())
+ .join("deps")
+ .join("save-analysis");
+
+ let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
+ tarball.include_target_in_component_name(true);
+ tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
+ Some(tarball.generate())
+ }
+}
+
+/// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
+/// `dst_dir`.
+fn copy_src_dirs(
+ builder: &Builder<'_>,
+ base: &Path,
+ src_dirs: &[&str],
+ exclude_dirs: &[&str],
+ dst_dir: &Path,
+) {
+ fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
+ let spath = match path.to_str() {
+ Some(path) => path,
+ None => return false,
+ };
+ if spath.ends_with('~') || spath.ends_with(".pyc") {
+ return false;
+ }
+
+ const LLVM_PROJECTS: &[&str] = &[
+ "llvm-project/clang",
+ "llvm-project\\clang",
+ "llvm-project/libunwind",
+ "llvm-project\\libunwind",
+ "llvm-project/lld",
+ "llvm-project\\lld",
+ "llvm-project/lldb",
+ "llvm-project\\lldb",
+ "llvm-project/llvm",
+ "llvm-project\\llvm",
+ "llvm-project/compiler-rt",
+ "llvm-project\\compiler-rt",
+ "llvm-project/cmake",
+ "llvm-project\\cmake",
+ ];
+ if spath.contains("llvm-project")
+ && !spath.ends_with("llvm-project")
+ && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
+ {
+ return false;
+ }
+
+ const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
+ if LLVM_TEST.iter().any(|path| spath.contains(path))
+ && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
+ {
+ return false;
+ }
+
+ let full_path = Path::new(dir).join(path);
+ if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
+ return false;
+ }
+
+ let excludes = [
+ "CVS",
+ "RCS",
+ "SCCS",
+ ".git",
+ ".gitignore",
+ ".gitmodules",
+ ".gitattributes",
+ ".cvsignore",
+ ".svn",
+ ".arch-ids",
+ "{arch}",
+ "=RELEASE-ID",
+ "=meta-update",
+ "=update",
+ ".bzr",
+ ".bzrignore",
+ ".bzrtags",
+ ".hg",
+ ".hgignore",
+ ".hgrags",
+ "_darcs",
+ ];
+ !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
+ }
+
+ // Copy the directories using our filter
+ for item in src_dirs {
+ let dst = &dst_dir.join(item);
+ t!(fs::create_dir_all(dst));
+ builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
+ }
+}
+
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Src;
+
+impl Step for Src {
+ /// The output path of the src installer tarball
+ type Output = GeneratedTarball;
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.alias("rust-src")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Src);
+ }
+
+ /// Creates the `rust-src` installer component
+ fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
+ let tarball = Tarball::new_targetless(builder, "rust-src");
+
+ // A lot of tools expect the rust-src component to be entirely in this directory, so if you
+ // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
+ // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
+ // and fix them...
+ //
+ // NOTE: if you update the paths here, you also should update the "virtual" path
+ // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
+ let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
+
+ let src_files = ["Cargo.lock"];
+ // This is the reduced set of paths which will become the rust-src component
+ // (essentially libstd and all of its path dependencies).
+ copy_src_dirs(
+ builder,
+ &builder.src,
+ &["library", "src/llvm-project/libunwind"],
+ &[
+ // not needed and contains symlinks which rustup currently
+ // chokes on when unpacking.
+ "library/backtrace/crates",
+ // these are 30MB combined and aren't necessary for building
+ // the standard library.
+ "library/stdarch/Cargo.toml",
+ "library/stdarch/crates/stdarch-verify",
+ "library/stdarch/crates/intrinsic-test",
+ ],
+ &dst_src,
+ );
+ for file in src_files.iter() {
+ builder.copy(&builder.src.join(file), &dst_src.join(file));
+ }
+
+ tarball.generate()
+ }
+}
+
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct PlainSourceTarball;
+
+impl Step for PlainSourceTarball {
+ /// Produces the location of the tarball generated
+ type Output = GeneratedTarball;
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let builder = run.builder;
+ run.alias("rustc-src").default_condition(builder.config.rust_dist_src)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(PlainSourceTarball);
+ }
+
+ /// Creates the plain source tarball
+ fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
+ // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which
+ // means neither rustup nor rustup-toolchain-install-master know how to download it.
+ // It also contains symbolic links, unlike other any other dist tarball.
+ // It's used for distros building rustc from source in a pre-vendored environment.
+ let mut tarball = Tarball::new(builder, "rustc", "src");
+ tarball.permit_symlinks(true);
+ let plain_dst_src = tarball.image_dir();
+
+ // This is the set of root paths which will become part of the source package
+ let src_files = [
+ "COPYRIGHT",
+ "LICENSE-APACHE",
+ "LICENSE-MIT",
+ "CONTRIBUTING.md",
+ "README.md",
+ "RELEASES.md",
+ "configure",
+ "x.py",
+ "config.toml.example",
+ "Cargo.toml",
+ "Cargo.lock",
+ ];
+ let src_dirs = ["src", "compiler", "library"];
+
+ copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
+
+ // Copy the files normally
+ for item in &src_files {
+ builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
+ }
+
+ // Create the version file
+ builder.create(&plain_dst_src.join("version"), &builder.rust_version());
+ if let Some(sha) = builder.rust_sha() {
+ builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
+ }
+
+ // If we're building from git sources, we need to vendor a complete distribution.
+ if builder.rust_info.is_git() {
+ // Ensure we have the submodules checked out.
+ builder.update_submodule(Path::new("src/tools/rust-analyzer"));
+
+ // Vendor all Cargo dependencies
+ let mut cmd = Command::new(&builder.initial_cargo);
+ cmd.arg("vendor")
+ .arg("--sync")
+ .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
+ .arg("--sync")
+ .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
+ .arg("--sync")
+ .arg(builder.src.join("./src/bootstrap/Cargo.toml"))
+ .current_dir(&plain_dst_src);
+
+ let config = if !builder.config.dry_run {
+ t!(String::from_utf8(t!(cmd.output()).stdout))
+ } else {
+ String::new()
+ };
+
+ let cargo_config_dir = plain_dst_src.join(".cargo");
+ builder.create_dir(&cargo_config_dir);
+ builder.create(&cargo_config_dir.join("config.toml"), &config);
+ }
+
+ tarball.bare()
+ }
+}
+
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Cargo {
+ pub compiler: Compiler,
+ pub target: TargetSelection,
+}
+
+impl Step for Cargo {
+ type Output = Option<GeneratedTarball>;
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let default = should_build_extended_tool(&run.builder, "cargo");
+ run.alias("cargo").default_condition(default)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Cargo {
+ compiler: run.builder.compiler_for(
+ run.builder.top_stage,
+ run.builder.config.build,
+ run.target,
+ ),
+ target: run.target,
+ });
+ }
+
+ fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
+ let compiler = self.compiler;
+ let target = self.target;
+
+ let cargo = builder.ensure(tool::Cargo { compiler, target });
+ let src = builder.src.join("src/tools/cargo");
+ let etc = src.join("src/etc");
+
+ // Prepare the image directory
+ let mut tarball = Tarball::new(builder, "cargo", &target.triple);
+ tarball.set_overlay(OverlayKind::Cargo);
+
+ tarball.add_file(&cargo, "bin", 0o755);
+ tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
+ tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
+ tarball.add_dir(etc.join("man"), "share/man/man1");
+ tarball.add_legal_and_readme_to("share/doc/cargo");
+
+ for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
+ let dirent = dirent.expect("read dir entry");
+ if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
+ tarball.add_file(&dirent.path(), "libexec", 0o755);
+ }
+ }
+
+ Some(tarball.generate())
+ }
+}
+
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Rls {
+ pub compiler: Compiler,
+ pub target: TargetSelection,
+}
+
+impl Step for Rls {
+ type Output = Option<GeneratedTarball>;
+ const ONLY_HOSTS: bool = true;
+ const DEFAULT: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let default = should_build_extended_tool(&run.builder, "rls");
+ run.alias("rls").default_condition(default)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Rls {
+ compiler: run.builder.compiler_for(
+ run.builder.top_stage,
+ run.builder.config.build,
+ run.target,
+ ),
+ target: run.target,
+ });
+ }
+
+ fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
+ let compiler = self.compiler;
+ let target = self.target;
+
+ let rls = builder
+ .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
+ .or_else(|| {
+ missing_tool("RLS", builder.build.config.missing_tools);
+ None
+ })?;
+
+ let mut tarball = Tarball::new(builder, "rls", &target.triple);
+ tarball.set_overlay(OverlayKind::RLS);
+ tarball.is_preview(true);
+ tarball.add_file(rls, "bin", 0o755);
+ tarball.add_legal_and_readme_to("share/doc/rls");
+ Some(tarball.generate())
+ }
+}
+
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RustAnalyzer {
+ pub compiler: Compiler,
+ pub target: TargetSelection,
+}
+
+impl Step for RustAnalyzer {
+ type Output = Option<GeneratedTarball>;
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let default = should_build_extended_tool(&run.builder, "rust-analyzer");
+ run.alias("rust-analyzer").default_condition(default)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(RustAnalyzer {
+ compiler: run.builder.compiler_for(
+ run.builder.top_stage,
+ run.builder.config.build,
+ run.target,
+ ),
+ target: run.target,
+ });
+ }
+
+ fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
+ let compiler = self.compiler;
+ let target = self.target;
+
+ if target.contains("riscv64") {
+ // riscv64 currently has an LLVM bug that makes rust-analyzer unable
+ // to build. See #74813 for details.
+ return None;
+ }
+
+ let rust_analyzer = builder
+ .ensure(tool::RustAnalyzer { compiler, target })
+ .expect("rust-analyzer always builds");
+
+ let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
+ tarball.set_overlay(OverlayKind::RustAnalyzer);
+ tarball.is_preview(true);
+ tarball.add_file(rust_analyzer, "bin", 0o755);
+ tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
+ Some(tarball.generate())
+ }
+}
+
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Clippy {
+ pub compiler: Compiler,
+ pub target: TargetSelection,
+}
+
+impl Step for Clippy {
+ type Output = Option<GeneratedTarball>;
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let default = should_build_extended_tool(&run.builder, "clippy");
+ run.alias("clippy").default_condition(default)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Clippy {
+ compiler: run.builder.compiler_for(
+ run.builder.top_stage,
+ run.builder.config.build,
+ run.target,
+ ),
+ target: run.target,
+ });
+ }
+
+ fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
+ let compiler = self.compiler;
+ let target = self.target;
+
+ // Prepare the image directory
+ // We expect clippy to build, because we've exited this step above if tool
+ // state for clippy isn't testing.
+ let clippy = builder
+ .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
+ .expect("clippy expected to build - essential tool");
+ let cargoclippy = builder
+ .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
+ .expect("clippy expected to build - essential tool");
+
+ let mut tarball = Tarball::new(builder, "clippy", &target.triple);
+ tarball.set_overlay(OverlayKind::Clippy);
+ tarball.is_preview(true);
+ tarball.add_file(clippy, "bin", 0o755);
+ tarball.add_file(cargoclippy, "bin", 0o755);
+ tarball.add_legal_and_readme_to("share/doc/clippy");
+ Some(tarball.generate())
+ }
+}
+
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Miri {
+ pub compiler: Compiler,
+ pub target: TargetSelection,
+}
+
+impl Step for Miri {
+ type Output = Option<GeneratedTarball>;
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let default = should_build_extended_tool(&run.builder, "miri");
+ run.alias("miri").default_condition(default)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Miri {
+ compiler: run.builder.compiler_for(
+ run.builder.top_stage,
+ run.builder.config.build,
+ run.target,
+ ),
+ target: run.target,
+ });
+ }
+
+ fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
+ // This prevents miri from being built for "dist" or "install"
+ // on the stable/beta channels. It is a nightly-only tool and should
+ // not be included.
+ if !builder.build.unstable_features() {
+ return None;
+ }
+ let compiler = self.compiler;
+ let target = self.target;
+
+ let miri = builder
+ .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
+ .or_else(|| {
+ missing_tool("miri", builder.build.config.missing_tools);
+ None
+ })?;
+ let cargomiri = builder
+ .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
+ .or_else(|| {
+ missing_tool("cargo miri", builder.build.config.missing_tools);
+ None
+ })?;
+
+ let mut tarball = Tarball::new(builder, "miri", &target.triple);
+ tarball.set_overlay(OverlayKind::Miri);
+ tarball.is_preview(true);
+ tarball.add_file(miri, "bin", 0o755);
+ tarball.add_file(cargomiri, "bin", 0o755);
+ tarball.add_legal_and_readme_to("share/doc/miri");
+ Some(tarball.generate())
+ }
+}
+
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Rustfmt {
+ pub compiler: Compiler,
+ pub target: TargetSelection,
+}
+
+impl Step for Rustfmt {
+ type Output = Option<GeneratedTarball>;
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let default = should_build_extended_tool(&run.builder, "rustfmt");
+ run.alias("rustfmt").default_condition(default)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Rustfmt {
+ compiler: run.builder.compiler_for(
+ run.builder.top_stage,
+ run.builder.config.build,
+ run.target,
+ ),
+ target: run.target,
+ });
+ }
+
+ fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
+ let compiler = self.compiler;
+ let target = self.target;
+
+ let rustfmt = builder
+ .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
+ .or_else(|| {
+ missing_tool("Rustfmt", builder.build.config.missing_tools);
+ None
+ })?;
+ let cargofmt = builder
+ .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
+ .or_else(|| {
+ missing_tool("Cargofmt", builder.build.config.missing_tools);
+ None
+ })?;
+
+ let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
+ tarball.set_overlay(OverlayKind::Rustfmt);
+ tarball.is_preview(true);
+ tarball.add_file(rustfmt, "bin", 0o755);
+ tarball.add_file(cargofmt, "bin", 0o755);
+ tarball.add_legal_and_readme_to("share/doc/rustfmt");
+ Some(tarball.generate())
+ }
+}
+
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RustDemangler {
+ pub compiler: Compiler,
+ pub target: TargetSelection,
+}
+
+impl Step for RustDemangler {
+ type Output = Option<GeneratedTarball>;
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ // While other tools use `should_build_extended_tool` to decide whether to be run by
+ // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
+ // the other ones or if `profiler = true`. Because we don't know the target at this stage
+ // we run the step by default when only `extended = true`, and decide whether to actually
+ // run it or not later.
+ let default = run.builder.config.extended;
+ run.alias("rust-demangler").default_condition(default)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(RustDemangler {
+ compiler: run.builder.compiler_for(
+ run.builder.top_stage,
+ run.builder.config.build,
+ run.target,
+ ),
+ target: run.target,
+ });
+ }
+
+ fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
+ let compiler = self.compiler;
+ let target = self.target;
+
+ // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
+ let condition = should_build_extended_tool(builder, "rust-demangler")
+ || builder.config.profiler_enabled(target);
+ if builder.config.extended && !condition {
+ return None;
+ }
+
+ let rust_demangler = builder
+ .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
+ .expect("rust-demangler expected to build - in-tree tool");
+
+ // Prepare the image directory
+ let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
+ tarball.set_overlay(OverlayKind::RustDemangler);
+ tarball.is_preview(true);
+ tarball.add_file(&rust_demangler, "bin", 0o755);
+ tarball.add_legal_and_readme_to("share/doc/rust-demangler");
+ Some(tarball.generate())
+ }
+}
+
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Extended {
+ stage: u32,
+ host: TargetSelection,
+ target: TargetSelection,
+}
+
+impl Step for Extended {
+ type Output = ();
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let builder = run.builder;
+ run.alias("extended").default_condition(builder.config.extended)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Extended {
+ stage: run.builder.top_stage,
+ host: run.builder.config.build,
+ target: run.target,
+ });
+ }
+
+ /// Creates a combined installer for the specified target in the provided stage.
+ fn run(self, builder: &Builder<'_>) {
+ let target = self.target;
+ let stage = self.stage;
+ let compiler = builder.compiler_for(self.stage, self.host, self.target);
+
+ builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
+
+ let mut tarballs = Vec::new();
+ let mut built_tools = HashSet::new();
+ macro_rules! add_component {
+ ($name:expr => $step:expr) => {
+ if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
+ tarballs.push(tarball);
+ built_tools.insert($name);
+ }
+ };
+ }
+
+ // When rust-std package split from rustc, we needed to ensure that during
+ // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
+ // the std files during uninstall. To do this ensure that rustc comes
+ // before rust-std in the list below.
+ tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
+ tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
+
+ if target.ends_with("windows-gnu") {
+ tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
+ }
+
+ add_component!("rust-docs" => Docs { host: target });
+ add_component!("rust-demangler"=> RustDemangler { compiler, target });
+ add_component!("cargo" => Cargo { compiler, target });
+ add_component!("rustfmt" => Rustfmt { compiler, target });
+ add_component!("rls" => Rls { compiler, target });
+ add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
+ add_component!("llvm-components" => LlvmTools { target });
+ add_component!("clippy" => Clippy { compiler, target });
+ add_component!("miri" => Miri { compiler, target });
+ add_component!("analysis" => Analysis { compiler, target });
+
+ let etc = builder.src.join("src/etc/installer");
+
+ // Avoid producing tarballs during a dry run.
+ if builder.config.dry_run {
+ return;
+ }
+
+ let tarball = Tarball::new(builder, "rust", &target.triple);
+ let generated = tarball.combine(&tarballs);
+
+ let tmp = tmpdir(builder).join("combined-tarball");
+ let work = generated.work_dir();
+
+ let mut license = String::new();
+ license += &builder.read(&builder.src.join("COPYRIGHT"));
+ license += &builder.read(&builder.src.join("LICENSE-APACHE"));
+ license += &builder.read(&builder.src.join("LICENSE-MIT"));
+ license.push('\n');
+ license.push('\n');
+
+ let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
+ let mut rtf = rtf.to_string();
+ rtf.push('\n');
+ for line in license.lines() {
+ rtf.push_str(line);
+ rtf.push_str("\\line ");
+ }
+ rtf.push('}');
+
+ fn filter(contents: &str, marker: &str) -> String {
+ let start = format!("tool-{}-start", marker);
+ let end = format!("tool-{}-end", marker);
+ let mut lines = Vec::new();
+ let mut omitted = false;
+ for line in contents.lines() {
+ if line.contains(&start) {
+ omitted = true;
+ } else if line.contains(&end) {
+ omitted = false;
+ } else if !omitted {
+ lines.push(line);
+ }
+ }
+
+ lines.join("\n")
+ }
+
+ let xform = |p: &Path| {
+ let mut contents = t!(fs::read_to_string(p));
+ for tool in &["rust-demangler", "rls", "rust-analyzer", "miri", "rustfmt"] {
+ if !built_tools.contains(tool) {
+ contents = filter(&contents, tool);
+ }
+ }
+ let ret = tmp.join(p.file_name().unwrap());
+ t!(fs::write(&ret, &contents));
+ ret
+ };
+
+ if target.contains("apple-darwin") {
+ builder.info("building pkg installer");
+ let pkg = tmp.join("pkg");
+ let _ = fs::remove_dir_all(&pkg);
+
+ let pkgbuild = |component: &str| {
+ let mut cmd = Command::new("pkgbuild");
+ cmd.arg("--identifier")
+ .arg(format!("org.rust-lang.{}", component))
+ .arg("--scripts")
+ .arg(pkg.join(component))
+ .arg("--nopayload")
+ .arg(pkg.join(component).with_extension("pkg"));
+ builder.run(&mut cmd);
+ };
+
+ let prepare = |name: &str| {
+ builder.create_dir(&pkg.join(name));
+ builder.cp_r(
+ &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
+ &pkg.join(name),
+ );
+ builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
+ pkgbuild(name);
+ };
+ prepare("rustc");
+ prepare("cargo");
+ prepare("rust-std");
+ prepare("rust-analysis");
+ prepare("clippy");
+ for tool in &["rust-docs", "rust-demangler", "rls", "rust-analyzer", "miri"] {
+ if built_tools.contains(tool) {
+ prepare(tool);
+ }
+ }
+ // create an 'uninstall' package
+ builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
+ pkgbuild("uninstall");
+
+ builder.create_dir(&pkg.join("res"));
+ builder.create(&pkg.join("res/LICENSE.txt"), &license);
+ builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
+ let mut cmd = Command::new("productbuild");
+ cmd.arg("--distribution")
+ .arg(xform(&etc.join("pkg/Distribution.xml")))
+ .arg("--resources")
+ .arg(pkg.join("res"))
+ .arg(distdir(builder).join(format!(
+ "{}-{}.pkg",
+ pkgname(builder, "rust"),
+ target.triple
+ )))
+ .arg("--package-path")
+ .arg(&pkg);
+ let _time = timeit(builder);
+ builder.run(&mut cmd);
+ }
+
+ if target.contains("windows") {
+ let exe = tmp.join("exe");
+ let _ = fs::remove_dir_all(&exe);
+
+ let prepare = |name: &str| {
+ builder.create_dir(&exe.join(name));
+ let dir = if name == "rust-std" || name == "rust-analysis" {
+ format!("{}-{}", name, target.triple)
+ } else if name == "rls" {
+ "rls-preview".to_string()
+ } else if name == "rust-analyzer" {
+ "rust-analyzer-preview".to_string()
+ } else if name == "clippy" {
+ "clippy-preview".to_string()
+ } else if name == "rust-demangler" {
+ "rust-demangler-preview".to_string()
+ } else if name == "miri" {
+ "miri-preview".to_string()
+ } else {
+ name.to_string()
+ };
+ builder.cp_r(
+ &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
+ &exe.join(name),
+ );
+ builder.remove(&exe.join(name).join("manifest.in"));
+ };
+ prepare("rustc");
+ prepare("cargo");
+ prepare("rust-analysis");
+ prepare("rust-docs");
+ prepare("rust-std");
+ prepare("clippy");
+ for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
+ if built_tools.contains(tool) {
+ prepare(tool);
+ }
+ }
+ if target.ends_with("windows-gnu") {
+ prepare("rust-mingw");
+ }
+
+ builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
+
+ // Generate msi installer
+ let wix_path = env::var_os("WIX")
+ .expect("`WIX` environment variable must be set for generating MSI installer(s).");
+ let wix = PathBuf::from(wix_path);
+ let heat = wix.join("bin/heat.exe");
+ let candle = wix.join("bin/candle.exe");
+ let light = wix.join("bin/light.exe");
+
+ let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
+ builder.run(
+ Command::new(&heat)
+ .current_dir(&exe)
+ .arg("dir")
+ .arg("rustc")
+ .args(&heat_flags)
+ .arg("-cg")
+ .arg("RustcGroup")
+ .arg("-dr")
+ .arg("Rustc")
+ .arg("-var")
+ .arg("var.RustcDir")
+ .arg("-out")
+ .arg(exe.join("RustcGroup.wxs")),
+ );
+ builder.run(
+ Command::new(&heat)
+ .current_dir(&exe)
+ .arg("dir")
+ .arg("rust-docs")
+ .args(&heat_flags)
+ .arg("-cg")
+ .arg("DocsGroup")
+ .arg("-dr")
+ .arg("Docs")
+ .arg("-var")
+ .arg("var.DocsDir")
+ .arg("-out")
+ .arg(exe.join("DocsGroup.wxs"))
+ .arg("-t")
+ .arg(etc.join("msi/squash-components.xsl")),
+ );
+ builder.run(
+ Command::new(&heat)
+ .current_dir(&exe)
+ .arg("dir")
+ .arg("cargo")
+ .args(&heat_flags)
+ .arg("-cg")
+ .arg("CargoGroup")
+ .arg("-dr")
+ .arg("Cargo")
+ .arg("-var")
+ .arg("var.CargoDir")
+ .arg("-out")
+ .arg(exe.join("CargoGroup.wxs"))
+ .arg("-t")
+ .arg(etc.join("msi/remove-duplicates.xsl")),
+ );
+ builder.run(
+ Command::new(&heat)
+ .current_dir(&exe)
+ .arg("dir")
+ .arg("rust-std")
+ .args(&heat_flags)
+ .arg("-cg")
+ .arg("StdGroup")
+ .arg("-dr")
+ .arg("Std")
+ .arg("-var")
+ .arg("var.StdDir")
+ .arg("-out")
+ .arg(exe.join("StdGroup.wxs")),
+ );
+ if built_tools.contains("rls") {
+ builder.run(
+ Command::new(&heat)
+ .current_dir(&exe)
+ .arg("dir")
+ .arg("rls")
+ .args(&heat_flags)
+ .arg("-cg")
+ .arg("RlsGroup")
+ .arg("-dr")
+ .arg("Rls")
+ .arg("-var")
+ .arg("var.RlsDir")
+ .arg("-out")
+ .arg(exe.join("RlsGroup.wxs"))
+ .arg("-t")
+ .arg(etc.join("msi/remove-duplicates.xsl")),
+ );
+ }
+ if built_tools.contains("rust-analyzer") {
+ builder.run(
+ Command::new(&heat)
+ .current_dir(&exe)
+ .arg("dir")
+ .arg("rust-analyzer")
+ .args(&heat_flags)
+ .arg("-cg")
+ .arg("RustAnalyzerGroup")
+ .arg("-dr")
+ .arg("RustAnalyzer")
+ .arg("-var")
+ .arg("var.RustAnalyzerDir")
+ .arg("-out")
+ .arg(exe.join("RustAnalyzerGroup.wxs"))
+ .arg("-t")
+ .arg(etc.join("msi/remove-duplicates.xsl")),
+ );
+ }
+ builder.run(
+ Command::new(&heat)
+ .current_dir(&exe)
+ .arg("dir")
+ .arg("clippy")
+ .args(&heat_flags)
+ .arg("-cg")
+ .arg("ClippyGroup")
+ .arg("-dr")
+ .arg("Clippy")
+ .arg("-var")
+ .arg("var.ClippyDir")
+ .arg("-out")
+ .arg(exe.join("ClippyGroup.wxs"))
+ .arg("-t")
+ .arg(etc.join("msi/remove-duplicates.xsl")),
+ );
+ if built_tools.contains("rust-demangler") {
+ builder.run(
+ Command::new(&heat)
+ .current_dir(&exe)
+ .arg("dir")
+ .arg("rust-demangler")
+ .args(&heat_flags)
+ .arg("-cg")
+ .arg("RustDemanglerGroup")
+ .arg("-dr")
+ .arg("RustDemangler")
+ .arg("-var")
+ .arg("var.RustDemanglerDir")
+ .arg("-out")
+ .arg(exe.join("RustDemanglerGroup.wxs"))
+ .arg("-t")
+ .arg(etc.join("msi/remove-duplicates.xsl")),
+ );
+ }
+ if built_tools.contains("miri") {
+ builder.run(
+ Command::new(&heat)
+ .current_dir(&exe)
+ .arg("dir")
+ .arg("miri")
+ .args(&heat_flags)
+ .arg("-cg")
+ .arg("MiriGroup")
+ .arg("-dr")
+ .arg("Miri")
+ .arg("-var")
+ .arg("var.MiriDir")
+ .arg("-out")
+ .arg(exe.join("MiriGroup.wxs"))
+ .arg("-t")
+ .arg(etc.join("msi/remove-duplicates.xsl")),
+ );
+ }
+ builder.run(
+ Command::new(&heat)
+ .current_dir(&exe)
+ .arg("dir")
+ .arg("rust-analysis")
+ .args(&heat_flags)
+ .arg("-cg")
+ .arg("AnalysisGroup")
+ .arg("-dr")
+ .arg("Analysis")
+ .arg("-var")
+ .arg("var.AnalysisDir")
+ .arg("-out")
+ .arg(exe.join("AnalysisGroup.wxs"))
+ .arg("-t")
+ .arg(etc.join("msi/remove-duplicates.xsl")),
+ );
+ if target.ends_with("windows-gnu") {
+ builder.run(
+ Command::new(&heat)
+ .current_dir(&exe)
+ .arg("dir")
+ .arg("rust-mingw")
+ .args(&heat_flags)
+ .arg("-cg")
+ .arg("GccGroup")
+ .arg("-dr")
+ .arg("Gcc")
+ .arg("-var")
+ .arg("var.GccDir")
+ .arg("-out")
+ .arg(exe.join("GccGroup.wxs")),
+ );
+ }
+
+ let candle = |input: &Path| {
+ let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
+ let arch = if target.contains("x86_64") { "x64" } else { "x86" };
+ let mut cmd = Command::new(&candle);
+ cmd.current_dir(&exe)
+ .arg("-nologo")
+ .arg("-dRustcDir=rustc")
+ .arg("-dDocsDir=rust-docs")
+ .arg("-dCargoDir=cargo")
+ .arg("-dStdDir=rust-std")
+ .arg("-dAnalysisDir=rust-analysis")
+ .arg("-dClippyDir=clippy")
+ .arg("-arch")
+ .arg(&arch)
+ .arg("-out")
+ .arg(&output)
+ .arg(&input);
+ add_env(builder, &mut cmd, target);
+
+ if built_tools.contains("rust-demangler") {
+ cmd.arg("-dRustDemanglerDir=rust-demangler");
+ }
+ if built_tools.contains("rls") {
+ cmd.arg("-dRlsDir=rls");
+ }
+ if built_tools.contains("rust-analyzer") {
+ cmd.arg("-dRustAnalyzerDir=rust-analyzer");
+ }
+ if built_tools.contains("miri") {
+ cmd.arg("-dMiriDir=miri");
+ }
+ if target.ends_with("windows-gnu") {
+ cmd.arg("-dGccDir=rust-mingw");
+ }
+ builder.run(&mut cmd);
+ };
+ candle(&xform(&etc.join("msi/rust.wxs")));
+ candle(&etc.join("msi/ui.wxs"));
+ candle(&etc.join("msi/rustwelcomedlg.wxs"));
+ candle("RustcGroup.wxs".as_ref());
+ candle("DocsGroup.wxs".as_ref());
+ candle("CargoGroup.wxs".as_ref());
+ candle("StdGroup.wxs".as_ref());
+ candle("ClippyGroup.wxs".as_ref());
+ if built_tools.contains("rust-demangler") {
+ candle("RustDemanglerGroup.wxs".as_ref());
+ }
+ if built_tools.contains("rls") {
+ candle("RlsGroup.wxs".as_ref());
+ }
+ if built_tools.contains("rust-analyzer") {
+ candle("RustAnalyzerGroup.wxs".as_ref());
+ }
+ if built_tools.contains("miri") {
+ candle("MiriGroup.wxs".as_ref());
+ }
+ candle("AnalysisGroup.wxs".as_ref());
+
+ if target.ends_with("windows-gnu") {
+ candle("GccGroup.wxs".as_ref());
+ }
+
+ builder.create(&exe.join("LICENSE.rtf"), &rtf);
+ builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
+ builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
+
+ builder.info(&format!("building `msi` installer with {:?}", light));
+ let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
+ let mut cmd = Command::new(&light);
+ cmd.arg("-nologo")
+ .arg("-ext")
+ .arg("WixUIExtension")
+ .arg("-ext")
+ .arg("WixUtilExtension")
+ .arg("-out")
+ .arg(exe.join(&filename))
+ .arg("rust.wixobj")
+ .arg("ui.wixobj")
+ .arg("rustwelcomedlg.wixobj")
+ .arg("RustcGroup.wixobj")
+ .arg("DocsGroup.wixobj")
+ .arg("CargoGroup.wixobj")
+ .arg("StdGroup.wixobj")
+ .arg("AnalysisGroup.wixobj")
+ .arg("ClippyGroup.wixobj")
+ .current_dir(&exe);
+
+ if built_tools.contains("rls") {
+ cmd.arg("RlsGroup.wixobj");
+ }
+ if built_tools.contains("rust-analyzer") {
+ cmd.arg("RustAnalyzerGroup.wixobj");
+ }
+ if built_tools.contains("rust-demangler") {
+ cmd.arg("RustDemanglerGroup.wixobj");
+ }
+ if built_tools.contains("miri") {
+ cmd.arg("MiriGroup.wixobj");
+ }
+
+ if target.ends_with("windows-gnu") {
+ cmd.arg("GccGroup.wixobj");
+ }
+ // ICE57 wrongly complains about the shortcuts
+ cmd.arg("-sice:ICE57");
+
+ let _time = timeit(builder);
+ builder.run(&mut cmd);
+
+ if !builder.config.dry_run {
+ t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
+ }
+ }
+ }
+}
+
+fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
+ let mut parts = builder.version.split('.');
+ cmd.env("CFG_RELEASE_INFO", builder.rust_version())
+ .env("CFG_RELEASE_NUM", &builder.version)
+ .env("CFG_RELEASE", builder.rust_release())
+ .env("CFG_VER_MAJOR", parts.next().unwrap())
+ .env("CFG_VER_MINOR", parts.next().unwrap())
+ .env("CFG_VER_PATCH", parts.next().unwrap())
+ .env("CFG_VER_BUILD", "0") // just needed to build
+ .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
+ .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
+ .env("CFG_BUILD", target.triple)
+ .env("CFG_CHANNEL", &builder.config.channel);
+
+ if target.contains("windows-gnullvm") {
+ cmd.env("CFG_MINGW", "1").env("CFG_ABI", "LLVM");
+ } else if target.contains("windows-gnu") {
+ cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
+ } else {
+ cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
+ }
+}
+
+/// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
+///
+/// Returns whether the files were actually copied.
+fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
+ if let Some(config) = builder.config.target_config.get(&target) {
+ if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
+ // If the LLVM was externally provided, then we don't currently copy
+ // artifacts into the sysroot. This is not necessarily the right
+ // choice (in particular, it will require the LLVM dylib to be in
+ // the linker's load path at runtime), but the common use case for
+ // external LLVMs is distribution provided LLVMs, and in that case
+ // they're usually in the standard search path (e.g., /usr/lib) and
+ // copying them here is going to cause problems as we may end up
+ // with the wrong files and isn't what distributions want.
+ //
+ // This behavior may be revisited in the future though.
+ //
+ // If the LLVM is coming from ourselves (just from CI) though, we
+ // still want to install it, as it otherwise won't be available.
+ return false;
+ }
+ }
+
+ // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
+ // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
+ // clear why this is the case, though. llvm-config will emit the versioned
+ // paths and we don't want those in the sysroot (as we're expecting
+ // unversioned paths).
+ if target.contains("apple-darwin") && builder.llvm_link_shared() {
+ let src_libdir = builder.llvm_out(target).join("lib");
+ let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
+ if llvm_dylib_path.exists() {
+ builder.install(&llvm_dylib_path, dst_libdir, 0o644);
+ }
+ !builder.config.dry_run
+ } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
+ let mut cmd = Command::new(llvm_config);
+ cmd.arg("--libfiles");
+ builder.verbose(&format!("running {:?}", cmd));
+ let files = output(&mut cmd);
+ let build_llvm_out = &builder.llvm_out(builder.config.build);
+ let target_llvm_out = &builder.llvm_out(target);
+ for file in files.trim_end().split(' ') {
+ // If we're not using a custom LLVM, make sure we package for the target.
+ let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
+ target_llvm_out.join(relative_path)
+ } else {
+ PathBuf::from(file)
+ };
+ builder.install(&file, dst_libdir, 0o644);
+ }
+ !builder.config.dry_run
+ } else {
+ false
+ }
+}
+
+/// Maybe add libLLVM.so to the target lib-dir for linking.
+pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
+ let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
+ // We do not need to copy LLVM files into the sysroot if it is not
+ // dynamically linked; it is already included into librustc_llvm
+ // statically.
+ if builder.llvm_link_shared() {
+ maybe_install_llvm(builder, target, &dst_libdir);
+ }
+}
+
+/// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
+pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
+ let dst_libdir =
+ sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
+ // We do not need to copy LLVM files into the sysroot if it is not
+ // dynamically linked; it is already included into librustc_llvm
+ // statically.
+ if builder.llvm_link_shared() {
+ maybe_install_llvm(builder, target, &dst_libdir);
+ }
+}
+
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+pub struct LlvmTools {
+ pub target: TargetSelection,
+}
+
+impl Step for LlvmTools {
+ type Output = Option<GeneratedTarball>;
+ const ONLY_HOSTS: bool = true;
+ const DEFAULT: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let default = should_build_extended_tool(&run.builder, "llvm-tools");
+ // FIXME: allow using the names of the tools themselves?
+ run.alias("llvm-tools").default_condition(default)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(LlvmTools { target: run.target });
+ }
+
+ fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
+ let target = self.target;
+
+ /* run only if llvm-config isn't used */
+ if let Some(config) = builder.config.target_config.get(&target) {
+ if let Some(ref _s) = config.llvm_config {
+ builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
+ return None;
+ }
+ }
+
+ builder.ensure(crate::native::Llvm { target });
+
+ let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
+ tarball.set_overlay(OverlayKind::LLVM);
+ tarball.is_preview(true);
+
+ // Prepare the image directory
+ let src_bindir = builder.llvm_out(target).join("bin");
+ let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
+ for tool in LLVM_TOOLS {
+ let exe = src_bindir.join(exe(tool, target));
+ tarball.add_file(&exe, &dst_bindir, 0o755);
+ }
+
+ // Copy libLLVM.so to the target lib dir as well, so the RPATH like
+ // `$ORIGIN/../lib` can find it. It may also be used as a dependency
+ // of `rustc-dev` to support the inherited `-lLLVM` when using the
+ // compiler libraries.
+ maybe_install_llvm_target(builder, target, tarball.image_dir());
+
+ Some(tarball.generate())
+ }
+}
+
+// Tarball intended for internal consumption to ease rustc/std development.
+//
+// Should not be considered stable by end users.
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+pub struct RustDev {
+ pub target: TargetSelection,
+}
+
+impl Step for RustDev {
+ type Output = Option<GeneratedTarball>;
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.alias("rust-dev")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(RustDev { target: run.target });
+ }
+
+ fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
+ let target = self.target;
+
+ /* run only if llvm-config isn't used */
+ if let Some(config) = builder.config.target_config.get(&target) {
+ if let Some(ref _s) = config.llvm_config {
+ builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
+ return None;
+ }
+ }
+
+ let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
+ tarball.set_overlay(OverlayKind::LLVM);
+
+ builder.ensure(crate::native::Llvm { target });
+
+ let src_bindir = builder.llvm_out(target).join("bin");
+ // If updating this list, you likely want to change
+ // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users
+ // will not pick up the extra file until LLVM gets bumped.
+ for bin in &[
+ "llvm-config",
+ "llvm-ar",
+ "llvm-objdump",
+ "llvm-profdata",
+ "llvm-bcanalyzer",
+ "llvm-cov",
+ "llvm-dwp",
+ "llvm-nm",
+ "llvm-dwarfdump",
+ ] {
+ tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
+ }
+
+ // We don't build LLD on some platforms, so only add it if it exists
+ let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
+ if lld_path.exists() {
+ tarball.add_file(lld_path, "bin", 0o755);
+ }
+
+ tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
+
+ // Copy the include directory as well; needed mostly to build
+ // librustc_llvm properly (e.g., llvm-config.h is in here). But also
+ // just broadly useful to be able to link against the bundled LLVM.
+ tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
+
+ // Copy libLLVM.so to the target lib dir as well, so the RPATH like
+ // `$ORIGIN/../lib` can find it. It may also be used as a dependency
+ // of `rustc-dev` to support the inherited `-lLLVM` when using the
+ // compiler libraries.
+ let dst_libdir = tarball.image_dir().join("lib");
+ maybe_install_llvm(builder, target, &dst_libdir);
+ let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" };
+ t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
+
+ Some(tarball.generate())
+ }
+}
+
+/// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the
+/// release process to avoid cloning the monorepo and building stuff.
+///
+/// Should not be considered stable by end users.
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+pub struct BuildManifest {
+ pub target: TargetSelection,
+}
+
+impl Step for BuildManifest {
+ type Output = GeneratedTarball;
+ const DEFAULT: bool = false;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.alias("build-manifest")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(BuildManifest { target: run.target });
+ }
+
+ fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
+ let build_manifest = builder.tool_exe(Tool::BuildManifest);
+
+ let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
+ tarball.add_file(&build_manifest, "bin", 0o755);
+ tarball.generate()
+ }
+}
+
+/// Tarball containing artifacts necessary to reproduce the build of rustc.
+///
+/// Currently this is the PGO profile data.
+///
+/// Should not be considered stable by end users.
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+pub struct ReproducibleArtifacts {
+ pub target: TargetSelection,
+}
+
+impl Step for ReproducibleArtifacts {
+ type Output = Option<GeneratedTarball>;
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.alias("reproducible-artifacts")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(ReproducibleArtifacts { target: run.target });
+ }
+
+ fn run(self, builder: &Builder<'_>) -> Self::Output {
+ let mut added_anything = false;
+ let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
+ if let Some(path) = builder.config.rust_profile_use.as_ref() {
+ tarball.add_file(path, ".", 0o644);
+ added_anything = true;
+ }
+ if let Some(path) = builder.config.llvm_profile_use.as_ref() {
+ tarball.add_file(path, ".", 0o644);
+ added_anything = true;
+ }
+ if added_anything { Some(tarball.generate()) } else { None }
+ }
+}
diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs
new file mode 100644
index 000000000..2852442d0
--- /dev/null
+++ b/src/bootstrap/doc.rs
@@ -0,0 +1,932 @@
+//! Documentation generation for rustbuilder.
+//!
+//! This module implements generation for all bits and pieces of documentation
+//! for the Rust project. This notably includes suites like the rust book, the
+//! nomicon, rust by example, standalone documentation, etc.
+//!
+//! Everything here is basically just a shim around calling either `rustbook` or
+//! `rustdoc`.
+
+use std::fs;
+use std::io;
+use std::path::{Path, PathBuf};
+
+use crate::builder::{Builder, Compiler, Kind, RunConfig, ShouldRun, Step};
+use crate::cache::{Interned, INTERNER};
+use crate::compile;
+use crate::config::{Config, TargetSelection};
+use crate::tool::{self, prepare_tool_cargo, SourceType, Tool};
+use crate::util::{symlink_dir, t, up_to_date};
+use crate::Mode;
+
+macro_rules! submodule_helper {
+ ($path:expr, submodule) => {
+ $path
+ };
+ ($path:expr, submodule = $submodule:literal) => {
+ $submodule
+ };
+}
+
+macro_rules! book {
+ ($($name:ident, $path:expr, $book_name:expr $(, submodule $(= $submodule:literal)? )? ;)+) => {
+ $(
+ #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+ pub struct $name {
+ target: TargetSelection,
+ }
+
+ impl Step for $name {
+ type Output = ();
+ const DEFAULT: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let builder = run.builder;
+ run.path($path).default_condition(builder.config.docs)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure($name {
+ target: run.target,
+ });
+ }
+
+ fn run(self, builder: &Builder<'_>) {
+ $(
+ let path = Path::new(submodule_helper!( $path, submodule $( = $submodule )? ));
+ builder.update_submodule(&path);
+ )?
+ builder.ensure(RustbookSrc {
+ target: self.target,
+ name: INTERNER.intern_str($book_name),
+ src: INTERNER.intern_path(builder.src.join($path)),
+ })
+ }
+ }
+ )+
+ }
+}
+
+// NOTE: When adding a book here, make sure to ALSO build the book by
+// adding a build step in `src/bootstrap/builder.rs`!
+// NOTE: Make sure to add the corresponding submodule when adding a new book.
+// FIXME: Make checking for a submodule automatic somehow (maybe by having a list of all submodules
+// and checking against it?).
+book!(
+ CargoBook, "src/tools/cargo/src/doc", "cargo", submodule = "src/tools/cargo";
+ ClippyBook, "src/tools/clippy/book", "clippy";
+ EditionGuide, "src/doc/edition-guide", "edition-guide", submodule;
+ EmbeddedBook, "src/doc/embedded-book", "embedded-book", submodule;
+ Nomicon, "src/doc/nomicon", "nomicon", submodule;
+ Reference, "src/doc/reference", "reference", submodule;
+ RustByExample, "src/doc/rust-by-example", "rust-by-example", submodule;
+ RustdocBook, "src/doc/rustdoc", "rustdoc";
+);
+
+fn open(builder: &Builder<'_>, path: impl AsRef<Path>) {
+ if builder.config.dry_run || !builder.config.cmd.open() {
+ return;
+ }
+
+ let path = path.as_ref();
+ builder.info(&format!("Opening doc {}", path.display()));
+ if let Err(err) = opener::open(path) {
+ builder.info(&format!("{}\n", err));
+ }
+}
+
+// "library/std" -> ["library", "std"]
+//
+// Used for deciding whether a particular step is one requested by the user on
+// the `x.py doc` command line, which determines whether `--open` will open that
+// page.
+pub(crate) fn components_simplified(path: &PathBuf) -> Vec<&str> {
+ path.iter().map(|component| component.to_str().unwrap_or("???")).collect()
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct UnstableBook {
+ target: TargetSelection,
+}
+
+impl Step for UnstableBook {
+ type Output = ();
+ const DEFAULT: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let builder = run.builder;
+ run.path("src/doc/unstable-book").default_condition(builder.config.docs)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(UnstableBook { target: run.target });
+ }
+
+ fn run(self, builder: &Builder<'_>) {
+ builder.ensure(UnstableBookGen { target: self.target });
+ builder.ensure(RustbookSrc {
+ target: self.target,
+ name: INTERNER.intern_str("unstable-book"),
+ src: INTERNER.intern_path(builder.md_doc_out(self.target).join("unstable-book")),
+ })
+ }
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+struct RustbookSrc {
+ target: TargetSelection,
+ name: Interned<String>,
+ src: Interned<PathBuf>,
+}
+
+impl Step for RustbookSrc {
+ type Output = ();
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.never()
+ }
+
+ /// Invoke `rustbook` for `target` for the doc book `name` from the `src` path.
+ ///
+ /// This will not actually generate any documentation if the documentation has
+ /// already been generated.
+ fn run(self, builder: &Builder<'_>) {
+ let target = self.target;
+ let name = self.name;
+ let src = self.src;
+ let out = builder.doc_out(target);
+ t!(fs::create_dir_all(&out));
+
+ let out = out.join(name);
+ let index = out.join("index.html");
+ let rustbook = builder.tool_exe(Tool::Rustbook);
+ let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook);
+ if builder.config.dry_run || up_to_date(&src, &index) && up_to_date(&rustbook, &index) {
+ return;
+ }
+ builder.info(&format!("Rustbook ({}) - {}", target, name));
+ let _ = fs::remove_dir_all(&out);
+
+ builder.run(rustbook_cmd.arg("build").arg(&src).arg("-d").arg(out));
+ }
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct TheBook {
+ compiler: Compiler,
+ target: TargetSelection,
+}
+
+impl Step for TheBook {
+ type Output = ();
+ const DEFAULT: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let builder = run.builder;
+ run.path("src/doc/book").default_condition(builder.config.docs)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(TheBook {
+ compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
+ target: run.target,
+ });
+ }
+
+ /// Builds the book and associated stuff.
+ ///
+ /// We need to build:
+ ///
+ /// * Book
+ /// * Older edition redirects
+ /// * Version info and CSS
+ /// * Index page
+ /// * Redirect pages
+ fn run(self, builder: &Builder<'_>) {
+ let relative_path = Path::new("src").join("doc").join("book");
+ builder.update_submodule(&relative_path);
+
+ let compiler = self.compiler;
+ let target = self.target;
+
+ // build book
+ builder.ensure(RustbookSrc {
+ target,
+ name: INTERNER.intern_str("book"),
+ src: INTERNER.intern_path(builder.src.join(&relative_path)),
+ });
+
+ // building older edition redirects
+ for edition in &["first-edition", "second-edition", "2018-edition"] {
+ builder.ensure(RustbookSrc {
+ target,
+ name: INTERNER.intern_string(format!("book/{}", edition)),
+ src: INTERNER.intern_path(builder.src.join(&relative_path).join(edition)),
+ });
+ }
+
+ // build the version info page and CSS
+ builder.ensure(Standalone { compiler, target });
+
+ // build the redirect pages
+ builder.info(&format!("Documenting book redirect pages ({})", target));
+ for file in t!(fs::read_dir(builder.src.join(&relative_path).join("redirects"))) {
+ let file = t!(file);
+ let path = file.path();
+ let path = path.to_str().unwrap();
+
+ invoke_rustdoc(builder, compiler, target, path);
+ }
+
+ if builder.was_invoked_explicitly::<Self>(Kind::Doc) {
+ let out = builder.doc_out(target);
+ let index = out.join("book").join("index.html");
+ open(builder, &index);
+ }
+ }
+}
+
+fn invoke_rustdoc(
+ builder: &Builder<'_>,
+ compiler: Compiler,
+ target: TargetSelection,
+ markdown: &str,
+) {
+ let out = builder.doc_out(target);
+
+ let path = builder.src.join("src/doc").join(markdown);
+
+ let header = builder.src.join("src/doc/redirect.inc");
+ let footer = builder.src.join("src/doc/footer.inc");
+ let version_info = out.join("version_info.html");
+
+ let mut cmd = builder.rustdoc_cmd(compiler);
+
+ let out = out.join("book");
+
+ cmd.arg("--html-after-content")
+ .arg(&footer)
+ .arg("--html-before-content")
+ .arg(&version_info)
+ .arg("--html-in-header")
+ .arg(&header)
+ .arg("--markdown-no-toc")
+ .arg("--markdown-playground-url")
+ .arg("https://play.rust-lang.org/")
+ .arg("-o")
+ .arg(&out)
+ .arg(&path)
+ .arg("--markdown-css")
+ .arg("../rust.css");
+
+ if !builder.config.docs_minification {
+ cmd.arg("-Z").arg("unstable-options").arg("--disable-minification");
+ }
+
+ builder.run(&mut cmd);
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Standalone {
+ compiler: Compiler,
+ target: TargetSelection,
+}
+
+impl Step for Standalone {
+ type Output = ();
+ const DEFAULT: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let builder = run.builder;
+ run.path("src/doc").default_condition(builder.config.docs)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Standalone {
+ compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
+ target: run.target,
+ });
+ }
+
+ /// Generates all standalone documentation as compiled by the rustdoc in `stage`
+ /// for the `target` into `out`.
+ ///
+ /// This will list all of `src/doc` looking for markdown files and appropriately
+ /// perform transformations like substituting `VERSION`, `SHORT_HASH`, and
+ /// `STAMP` along with providing the various header/footer HTML we've customized.
+ ///
+ /// In the end, this is just a glorified wrapper around rustdoc!
+ fn run(self, builder: &Builder<'_>) {
+ let target = self.target;
+ let compiler = self.compiler;
+ builder.info(&format!("Documenting standalone ({})", target));
+ let out = builder.doc_out(target);
+ t!(fs::create_dir_all(&out));
+
+ let favicon = builder.src.join("src/doc/favicon.inc");
+ let footer = builder.src.join("src/doc/footer.inc");
+ let full_toc = builder.src.join("src/doc/full-toc.inc");
+ t!(fs::copy(builder.src.join("src/doc/rust.css"), out.join("rust.css")));
+
+ let version_input = builder.src.join("src/doc/version_info.html.template");
+ let version_info = out.join("version_info.html");
+
+ if !builder.config.dry_run && !up_to_date(&version_input, &version_info) {
+ let info = t!(fs::read_to_string(&version_input))
+ .replace("VERSION", &builder.rust_release())
+ .replace("SHORT_HASH", builder.rust_info.sha_short().unwrap_or(""))
+ .replace("STAMP", builder.rust_info.sha().unwrap_or(""));
+ t!(fs::write(&version_info, &info));
+ }
+
+ for file in t!(fs::read_dir(builder.src.join("src/doc"))) {
+ let file = t!(file);
+ let path = file.path();
+ let filename = path.file_name().unwrap().to_str().unwrap();
+ if !filename.ends_with(".md") || filename == "README.md" {
+ continue;
+ }
+
+ let html = out.join(filename).with_extension("html");
+ let rustdoc = builder.rustdoc(compiler);
+ if up_to_date(&path, &html)
+ && up_to_date(&footer, &html)
+ && up_to_date(&favicon, &html)
+ && up_to_date(&full_toc, &html)
+ && (builder.config.dry_run || up_to_date(&version_info, &html))
+ && (builder.config.dry_run || up_to_date(&rustdoc, &html))
+ {
+ continue;
+ }
+
+ let mut cmd = builder.rustdoc_cmd(compiler);
+ // Needed for --index-page flag
+ cmd.arg("-Z").arg("unstable-options");
+
+ cmd.arg("--html-after-content")
+ .arg(&footer)
+ .arg("--html-before-content")
+ .arg(&version_info)
+ .arg("--html-in-header")
+ .arg(&favicon)
+ .arg("--markdown-no-toc")
+ .arg("--index-page")
+ .arg(&builder.src.join("src/doc/index.md"))
+ .arg("--markdown-playground-url")
+ .arg("https://play.rust-lang.org/")
+ .arg("-o")
+ .arg(&out)
+ .arg(&path);
+
+ if !builder.config.docs_minification {
+ cmd.arg("--disable-minification");
+ }
+
+ if filename == "not_found.md" {
+ cmd.arg("--markdown-css")
+ .arg(format!("https://doc.rust-lang.org/rustdoc{}.css", &builder.version))
+ .arg("--markdown-css")
+ .arg("https://doc.rust-lang.org/rust.css");
+ } else {
+ cmd.arg("--markdown-css")
+ .arg(format!("rustdoc{}.css", &builder.version))
+ .arg("--markdown-css")
+ .arg("rust.css");
+ }
+ builder.run(&mut cmd);
+ }
+
+ // We open doc/index.html as the default if invoked as `x.py doc --open`
+ // with no particular explicit doc requested (e.g. library/core).
+ if builder.paths.is_empty() || builder.was_invoked_explicitly::<Self>(Kind::Doc) {
+ let index = out.join("index.html");
+ open(builder, &index);
+ }
+ }
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Std {
+ pub stage: u32,
+ pub target: TargetSelection,
+}
+
+impl Step for Std {
+ type Output = ();
+ const DEFAULT: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let builder = run.builder;
+ run.all_krates("test").path("library").default_condition(builder.config.docs)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Std { stage: run.builder.top_stage, target: run.target });
+ }
+
+ /// Compile all standard library documentation.
+ ///
+ /// This will generate all documentation for the standard library and its
+ /// dependencies. This is largely just a wrapper around `cargo doc`.
+ fn run(self, builder: &Builder<'_>) {
+ let stage = self.stage;
+ let target = self.target;
+ builder.info(&format!("Documenting stage{} std ({})", stage, target));
+ if builder.no_std(target) == Some(true) {
+ panic!(
+ "building std documentation for no_std target {target} is not supported\n\
+ Set `docs = false` in the config to disable documentation."
+ );
+ }
+ let out = builder.doc_out(target);
+ t!(fs::create_dir_all(&out));
+ let compiler = builder.compiler(stage, builder.config.build);
+
+ let out_dir = builder.stage_out(compiler, Mode::Std).join(target.triple).join("doc");
+
+ t!(fs::copy(builder.src.join("src/doc/rust.css"), out.join("rust.css")));
+
+ let run_cargo_rustdoc_for = |package: &str| {
+ let mut cargo =
+ builder.cargo(compiler, Mode::Std, SourceType::InTree, target, "rustdoc");
+ compile::std_cargo(builder, target, compiler.stage, &mut cargo);
+
+ cargo
+ .arg("-p")
+ .arg(package)
+ .arg("-Zskip-rustdoc-fingerprint")
+ .arg("--")
+ .arg("--markdown-css")
+ .arg("rust.css")
+ .arg("--markdown-no-toc")
+ .arg("-Z")
+ .arg("unstable-options")
+ .arg("--resource-suffix")
+ .arg(&builder.version)
+ .arg("--index-page")
+ .arg(&builder.src.join("src/doc/index.md"));
+
+ if !builder.config.docs_minification {
+ cargo.arg("--disable-minification");
+ }
+
+ builder.run(&mut cargo.into());
+ };
+
+ let paths = builder
+ .paths
+ .iter()
+ .map(components_simplified)
+ .filter_map(|path| {
+ if path.len() >= 2 && path.get(0) == Some(&"library") {
+ // single crate
+ Some(path[1].to_owned())
+ } else if !path.is_empty() {
+ // ??
+ Some(path[0].to_owned())
+ } else {
+ // all library crates
+ None
+ }
+ })
+ .collect::<Vec<_>>();
+
+ // Only build the following crates. While we could just iterate over the
+ // folder structure, that would also build internal crates that we do
+ // not want to show in documentation. These crates will later be visited
+ // by the rustc step, so internal documentation will show them.
+ //
+ // Note that the order here is important! The crates need to be
+ // processed starting from the leaves, otherwise rustdoc will not
+ // create correct links between crates because rustdoc depends on the
+ // existence of the output directories to know if it should be a local
+ // or remote link.
+ let krates = ["core", "alloc", "std", "proc_macro", "test"];
+ for krate in &krates {
+ run_cargo_rustdoc_for(krate);
+ if paths.iter().any(|p| p == krate) {
+ // No need to document more of the libraries if we have the one we want.
+ break;
+ }
+ }
+ builder.cp_r(&out_dir, &out);
+
+ // Look for library/std, library/core etc in the `x.py doc` arguments and
+ // open the corresponding rendered docs.
+ for requested_crate in paths {
+ if krates.iter().any(|k| *k == requested_crate.as_str()) {
+ let index = out.join(requested_crate).join("index.html");
+ open(builder, &index);
+ }
+ }
+ }
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Rustc {
+ pub stage: u32,
+ pub target: TargetSelection,
+}
+
+impl Step for Rustc {
+ type Output = ();
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let builder = run.builder;
+ run.crate_or_deps("rustc-main")
+ .path("compiler")
+ .default_condition(builder.config.compiler_docs)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Rustc { stage: run.builder.top_stage, target: run.target });
+ }
+
+ /// Generates compiler documentation.
+ ///
+ /// This will generate all documentation for compiler and dependencies.
+ /// Compiler documentation is distributed separately, so we make sure
+ /// we do not merge it with the other documentation from std, test and
+ /// proc_macros. This is largely just a wrapper around `cargo doc`.
+ fn run(self, builder: &Builder<'_>) {
+ let stage = self.stage;
+ let target = self.target;
+
+ let paths = builder
+ .paths
+ .iter()
+ .filter(|path| {
+ let components = components_simplified(path);
+ components.len() >= 2 && components[0] == "compiler"
+ })
+ .collect::<Vec<_>>();
+
+ // This is the intended out directory for compiler documentation.
+ let out = builder.compiler_doc_out(target);
+ t!(fs::create_dir_all(&out));
+
+ // Build the standard library, so that proc-macros can use it.
+ // (Normally, only the metadata would be necessary, but proc-macros are special since they run at compile-time.)
+ let compiler = builder.compiler(stage, builder.config.build);
+ builder.ensure(compile::Std::new(compiler, builder.config.build));
+
+ builder.info(&format!("Documenting stage{} compiler ({})", stage, target));
+
+ // This uses a shared directory so that librustdoc documentation gets
+ // correctly built and merged with the rustc documentation. This is
+ // needed because rustdoc is built in a different directory from
+ // rustc. rustdoc needs to be able to see everything, for example when
+ // merging the search index, or generating local (relative) links.
+ let out_dir = builder.stage_out(compiler, Mode::Rustc).join(target.triple).join("doc");
+ t!(symlink_dir_force(&builder.config, &out, &out_dir));
+ // Cargo puts proc macros in `target/doc` even if you pass `--target`
+ // explicitly (https://github.com/rust-lang/cargo/issues/7677).
+ let proc_macro_out_dir = builder.stage_out(compiler, Mode::Rustc).join("doc");
+ t!(symlink_dir_force(&builder.config, &out, &proc_macro_out_dir));
+
+ // Build cargo command.
+ let mut cargo = builder.cargo(compiler, Mode::Rustc, SourceType::InTree, target, "doc");
+ cargo.rustdocflag("--document-private-items");
+ // Since we always pass --document-private-items, there's no need to warn about linking to private items.
+ cargo.rustdocflag("-Arustdoc::private-intra-doc-links");
+ cargo.rustdocflag("--enable-index-page");
+ cargo.rustdocflag("-Zunstable-options");
+ cargo.rustdocflag("-Znormalize-docs");
+ cargo.rustdocflag("--show-type-layout");
+ cargo.rustdocflag("--generate-link-to-definition");
+ compile::rustc_cargo(builder, &mut cargo, target);
+ cargo.arg("-Zunstable-options");
+ cargo.arg("-Zskip-rustdoc-fingerprint");
+
+ // Only include compiler crates, no dependencies of those, such as `libc`.
+ // Do link to dependencies on `docs.rs` however using `rustdoc-map`.
+ cargo.arg("--no-deps");
+ cargo.arg("-Zrustdoc-map");
+
+ // FIXME: `-Zrustdoc-map` does not yet correctly work for transitive dependencies,
+ // once this is no longer an issue the special case for `ena` can be removed.
+ cargo.rustdocflag("--extern-html-root-url");
+ cargo.rustdocflag("ena=https://docs.rs/ena/latest/");
+
+ let root_crates = if paths.is_empty() {
+ vec![
+ INTERNER.intern_str("rustc_driver"),
+ INTERNER.intern_str("rustc_codegen_llvm"),
+ INTERNER.intern_str("rustc_codegen_ssa"),
+ ]
+ } else {
+ paths.into_iter().map(|p| builder.crate_paths[p]).collect()
+ };
+ // Find dependencies for top level crates.
+ let compiler_crates = root_crates.iter().flat_map(|krate| {
+ builder.in_tree_crates(krate, Some(target)).into_iter().map(|krate| krate.name)
+ });
+
+ let mut to_open = None;
+ for krate in compiler_crates {
+ // Create all crate output directories first to make sure rustdoc uses
+ // relative links.
+ // FIXME: Cargo should probably do this itself.
+ t!(fs::create_dir_all(out_dir.join(krate)));
+ cargo.arg("-p").arg(krate);
+ if to_open.is_none() {
+ to_open = Some(krate);
+ }
+ }
+
+ builder.run(&mut cargo.into());
+ // Let's open the first crate documentation page:
+ if let Some(krate) = to_open {
+ let index = out.join(krate).join("index.html");
+ open(builder, &index);
+ }
+ }
+}
+
+macro_rules! tool_doc {
+ ($tool: ident, $should_run: literal, $path: literal, [$($krate: literal),+ $(,)?], in_tree = $in_tree:expr $(,)?) => {
+ #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+ pub struct $tool {
+ target: TargetSelection,
+ }
+
+ impl Step for $tool {
+ type Output = ();
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let builder = run.builder;
+ run.crate_or_deps($should_run).default_condition(builder.config.compiler_docs)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure($tool { target: run.target });
+ }
+
+ /// Generates compiler documentation.
+ ///
+ /// This will generate all documentation for compiler and dependencies.
+ /// Compiler documentation is distributed separately, so we make sure
+ /// we do not merge it with the other documentation from std, test and
+ /// proc_macros. This is largely just a wrapper around `cargo doc`.
+ fn run(self, builder: &Builder<'_>) {
+ let stage = builder.top_stage;
+ let target = self.target;
+
+ // This is the intended out directory for compiler documentation.
+ let out = builder.compiler_doc_out(target);
+ t!(fs::create_dir_all(&out));
+
+ // Build rustc docs so that we generate relative links.
+ builder.ensure(Rustc { stage, target });
+ // Rustdoc needs the rustc sysroot available to build.
+ // FIXME: is there a way to only ensure `check::Rustc` here? Last time I tried it failed
+ // with strange errors, but only on a full bors test ...
+ let compiler = builder.compiler(stage, builder.config.build);
+ builder.ensure(compile::Rustc::new(compiler, target));
+
+ builder.info(
+ &format!(
+ "Documenting stage{} {} ({})",
+ stage,
+ stringify!($tool).to_lowercase(),
+ target,
+ ),
+ );
+
+ // Symlink compiler docs to the output directory of rustdoc documentation.
+ let out_dir = builder.stage_out(compiler, Mode::ToolRustc).join(target.triple).join("doc");
+ t!(fs::create_dir_all(&out_dir));
+ t!(symlink_dir_force(&builder.config, &out, &out_dir));
+
+ let source_type = if $in_tree == true {
+ SourceType::InTree
+ } else {
+ SourceType::Submodule
+ };
+
+ // Build cargo command.
+ let mut cargo = prepare_tool_cargo(
+ builder,
+ compiler,
+ Mode::ToolRustc,
+ target,
+ "doc",
+ $path,
+ source_type,
+ &[],
+ );
+
+ cargo.arg("-Zskip-rustdoc-fingerprint");
+ // Only include compiler crates, no dependencies of those, such as `libc`.
+ cargo.arg("--no-deps");
+ $(
+ cargo.arg("-p").arg($krate);
+ )+
+
+ cargo.rustdocflag("--document-private-items");
+ cargo.rustdocflag("--enable-index-page");
+ cargo.rustdocflag("--show-type-layout");
+ cargo.rustdocflag("--generate-link-to-definition");
+ cargo.rustdocflag("-Zunstable-options");
+ if $in_tree == true {
+ builder.run(&mut cargo.into());
+ } else {
+ // Allow out-of-tree docs to fail (since the tool might be in a broken state).
+ if !builder.try_run(&mut cargo.into()) {
+ builder.info(&format!(
+ "WARNING: tool {} failed to document; ignoring failure because it is an out-of-tree tool",
+ stringify!($tool).to_lowercase(),
+ ));
+ }
+ }
+ }
+ }
+ }
+}
+
+tool_doc!(
+ Rustdoc,
+ "rustdoc-tool",
+ "src/tools/rustdoc",
+ ["rustdoc", "rustdoc-json-types"],
+ in_tree = true
+);
+tool_doc!(
+ Rustfmt,
+ "rustfmt-nightly",
+ "src/tools/rustfmt",
+ ["rustfmt-nightly", "rustfmt-config_proc_macro"],
+ in_tree = true
+);
+tool_doc!(Clippy, "clippy", "src/tools/clippy", ["clippy_utils"], in_tree = true);
+tool_doc!(Miri, "miri", "src/tools/miri", ["miri"], in_tree = false);
+
+#[derive(Ord, PartialOrd, Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct ErrorIndex {
+ pub target: TargetSelection,
+}
+
+impl Step for ErrorIndex {
+ type Output = ();
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let builder = run.builder;
+ run.path("src/tools/error_index_generator").default_condition(builder.config.docs)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ let target = run.target;
+ run.builder.ensure(ErrorIndex { target });
+ }
+
+ /// Generates the HTML rendered error-index by running the
+ /// `error_index_generator` tool.
+ fn run(self, builder: &Builder<'_>) {
+ builder.info(&format!("Documenting error index ({})", self.target));
+ let out = builder.doc_out(self.target);
+ t!(fs::create_dir_all(&out));
+ let mut index = tool::ErrorIndex::command(builder);
+ index.arg("html");
+ index.arg(out.join("error-index.html"));
+ index.arg(&builder.version);
+
+ builder.run(&mut index);
+ }
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct UnstableBookGen {
+ target: TargetSelection,
+}
+
+impl Step for UnstableBookGen {
+ type Output = ();
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let builder = run.builder;
+ run.path("src/tools/unstable-book-gen").default_condition(builder.config.docs)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(UnstableBookGen { target: run.target });
+ }
+
+ fn run(self, builder: &Builder<'_>) {
+ let target = self.target;
+
+ builder.info(&format!("Generating unstable book md files ({})", target));
+ let out = builder.md_doc_out(target).join("unstable-book");
+ builder.create_dir(&out);
+ builder.remove_dir(&out);
+ let mut cmd = builder.tool_cmd(Tool::UnstableBookGen);
+ cmd.arg(builder.src.join("library"));
+ cmd.arg(builder.src.join("compiler"));
+ cmd.arg(builder.src.join("src"));
+ cmd.arg(out);
+
+ builder.run(&mut cmd);
+ }
+}
+
+fn symlink_dir_force(config: &Config, src: &Path, dst: &Path) -> io::Result<()> {
+ if config.dry_run {
+ return Ok(());
+ }
+ if let Ok(m) = fs::symlink_metadata(dst) {
+ if m.file_type().is_dir() {
+ fs::remove_dir_all(dst)?;
+ } else {
+ // handle directory junctions on windows by falling back to
+ // `remove_dir`.
+ fs::remove_file(dst).or_else(|_| fs::remove_dir(dst))?;
+ }
+ }
+
+ symlink_dir(config, src, dst)
+}
+
+#[derive(Ord, PartialOrd, Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RustcBook {
+ pub compiler: Compiler,
+ pub target: TargetSelection,
+ pub validate: bool,
+}
+
+impl Step for RustcBook {
+ type Output = ();
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let builder = run.builder;
+ run.path("src/doc/rustc").default_condition(builder.config.docs)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(RustcBook {
+ compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
+ target: run.target,
+ validate: false,
+ });
+ }
+
+ /// Builds the rustc book.
+ ///
+ /// The lints are auto-generated by a tool, and then merged into the book
+ /// in the "md-doc" directory in the build output directory. Then
+ /// "rustbook" is used to convert it to HTML.
+ fn run(self, builder: &Builder<'_>) {
+ let out_base = builder.md_doc_out(self.target).join("rustc");
+ t!(fs::create_dir_all(&out_base));
+ let out_listing = out_base.join("src/lints");
+ builder.cp_r(&builder.src.join("src/doc/rustc"), &out_base);
+ builder.info(&format!("Generating lint docs ({})", self.target));
+
+ let rustc = builder.rustc(self.compiler);
+ // The tool runs `rustc` for extracting output examples, so it needs a
+ // functional sysroot.
+ builder.ensure(compile::Std::new(self.compiler, self.target));
+ let mut cmd = builder.tool_cmd(Tool::LintDocs);
+ cmd.arg("--src");
+ cmd.arg(builder.src.join("compiler"));
+ cmd.arg("--out");
+ cmd.arg(&out_listing);
+ cmd.arg("--rustc");
+ cmd.arg(&rustc);
+ cmd.arg("--rustc-target").arg(&self.target.rustc_target_arg());
+ if builder.config.verbose() {
+ cmd.arg("--verbose");
+ }
+ if self.validate {
+ cmd.arg("--validate");
+ }
+ if !builder.unstable_features() {
+ // We need to validate nightly features, even on the stable channel.
+ cmd.env("RUSTC_BOOTSTRAP", "1");
+ }
+ // If the lib directories are in an unusual location (changed in
+ // config.toml), then this needs to explicitly update the dylib search
+ // path.
+ builder.add_rustc_lib_path(self.compiler, &mut cmd);
+ builder.run(&mut cmd);
+ // Run rustbook/mdbook to generate the HTML pages.
+ builder.ensure(RustbookSrc {
+ target: self.target,
+ name: INTERNER.intern_str("rustc"),
+ src: INTERNER.intern_path(out_base),
+ });
+ if builder.was_invoked_explicitly::<Self>(Kind::Doc) {
+ let out = builder.doc_out(self.target);
+ let index = out.join("rustc").join("index.html");
+ open(builder, &index);
+ }
+ }
+}
diff --git a/src/bootstrap/download-ci-llvm-stamp b/src/bootstrap/download-ci-llvm-stamp
new file mode 100644
index 000000000..19504a51a
--- /dev/null
+++ b/src/bootstrap/download-ci-llvm-stamp
@@ -0,0 +1,4 @@
+Change this file to make users of the `download-ci-llvm` configuration download
+a new version of LLVM from CI, even if the LLVM submodule hasn’t changed.
+
+Last change is for: https://github.com/rust-lang/rust/pull/96867
diff --git a/src/bootstrap/dylib_util.rs b/src/bootstrap/dylib_util.rs
new file mode 100644
index 000000000..6d75272c5
--- /dev/null
+++ b/src/bootstrap/dylib_util.rs
@@ -0,0 +1,28 @@
+// Various utilities for working with dylib paths.
+//
+// This file is meant to be included directly to avoid a dependency on the bootstrap library from
+// the rustc and rustdoc wrappers. This improves compilation time by reducing the linking time.
+
+/// Returns the environment variable which the dynamic library lookup path
+/// resides in for this platform.
+pub fn dylib_path_var() -> &'static str {
+ if cfg!(target_os = "windows") {
+ "PATH"
+ } else if cfg!(target_os = "macos") {
+ "DYLD_LIBRARY_PATH"
+ } else if cfg!(target_os = "haiku") {
+ "LIBRARY_PATH"
+ } else {
+ "LD_LIBRARY_PATH"
+ }
+}
+
+/// Parses the `dylib_path_var()` environment variable, returning a list of
+/// paths that are members of this lookup path.
+pub fn dylib_path() -> Vec<PathBuf> {
+ let var = match env::var_os(dylib_path_var()) {
+ Some(v) => v,
+ None => return vec![],
+ };
+ env::split_paths(&var).collect()
+}
diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs
new file mode 100644
index 000000000..80b3bcce8
--- /dev/null
+++ b/src/bootstrap/flags.rs
@@ -0,0 +1,817 @@
+//! Command-line interface of the rustbuild build system.
+//!
+//! This module implements the command-line parsing of the build system which
+//! has various flags to configure how it's run.
+
+use std::path::PathBuf;
+
+use getopts::Options;
+
+use crate::builder::{Builder, Kind};
+use crate::config::{Config, TargetSelection};
+use crate::setup::Profile;
+use crate::util::t;
+use crate::{Build, DocTests};
+
+#[derive(Copy, Clone)]
+pub enum Color {
+ Always,
+ Never,
+ Auto,
+}
+
+impl Default for Color {
+ fn default() -> Self {
+ Self::Auto
+ }
+}
+
+impl std::str::FromStr for Color {
+ type Err = ();
+
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
+ match s.to_lowercase().as_str() {
+ "always" => Ok(Self::Always),
+ "never" => Ok(Self::Never),
+ "auto" => Ok(Self::Auto),
+ _ => Err(()),
+ }
+ }
+}
+
+/// Deserialized version of all flags for this compile.
+pub struct Flags {
+ pub verbose: usize, // number of -v args; each extra -v after the first is passed to Cargo
+ pub on_fail: Option<String>,
+ pub stage: Option<u32>,
+ pub keep_stage: Vec<u32>,
+ pub keep_stage_std: Vec<u32>,
+
+ pub host: Option<Vec<TargetSelection>>,
+ pub target: Option<Vec<TargetSelection>>,
+ pub config: Option<PathBuf>,
+ pub build_dir: Option<PathBuf>,
+ pub jobs: Option<u32>,
+ pub cmd: Subcommand,
+ pub incremental: bool,
+ pub exclude: Vec<PathBuf>,
+ pub include_default_paths: bool,
+ pub rustc_error_format: Option<String>,
+ pub json_output: bool,
+ pub dry_run: bool,
+ pub color: Color,
+
+ // This overrides the deny-warnings configuration option,
+ // which passes -Dwarnings to the compiler invocations.
+ //
+ // true => deny, false => warn
+ pub deny_warnings: Option<bool>,
+
+ pub llvm_skip_rebuild: Option<bool>,
+
+ pub rust_profile_use: Option<String>,
+ pub rust_profile_generate: Option<String>,
+
+ pub llvm_profile_use: Option<String>,
+ // LLVM doesn't support a custom location for generating profile
+ // information.
+ //
+ // llvm_out/build/profiles/ is the location this writes to.
+ pub llvm_profile_generate: bool,
+}
+
+#[cfg_attr(test, derive(Clone))]
+pub enum Subcommand {
+ Build {
+ paths: Vec<PathBuf>,
+ },
+ Check {
+ paths: Vec<PathBuf>,
+ },
+ Clippy {
+ fix: bool,
+ paths: Vec<PathBuf>,
+ clippy_lint_allow: Vec<String>,
+ clippy_lint_deny: Vec<String>,
+ clippy_lint_warn: Vec<String>,
+ clippy_lint_forbid: Vec<String>,
+ },
+ Fix {
+ paths: Vec<PathBuf>,
+ },
+ Format {
+ paths: Vec<PathBuf>,
+ check: bool,
+ },
+ Doc {
+ paths: Vec<PathBuf>,
+ open: bool,
+ },
+ Test {
+ paths: Vec<PathBuf>,
+ /// Whether to automatically update stderr/stdout files
+ bless: bool,
+ force_rerun: bool,
+ compare_mode: Option<String>,
+ pass: Option<String>,
+ run: Option<String>,
+ skip: Vec<String>,
+ test_args: Vec<String>,
+ rustc_args: Vec<String>,
+ fail_fast: bool,
+ doc_tests: DocTests,
+ rustfix_coverage: bool,
+ },
+ Bench {
+ paths: Vec<PathBuf>,
+ test_args: Vec<String>,
+ },
+ Clean {
+ all: bool,
+ },
+ Dist {
+ paths: Vec<PathBuf>,
+ },
+ Install {
+ paths: Vec<PathBuf>,
+ },
+ Run {
+ paths: Vec<PathBuf>,
+ },
+ Setup {
+ profile: Profile,
+ },
+}
+
+impl Default for Subcommand {
+ fn default() -> Subcommand {
+ Subcommand::Build { paths: vec![PathBuf::from("nowhere")] }
+ }
+}
+
+impl Flags {
+ pub fn parse(args: &[String]) -> Flags {
+ let mut subcommand_help = String::from(
+ "\
+Usage: x.py <subcommand> [options] [<paths>...]
+
+Subcommands:
+ build, b Compile either the compiler or libraries
+ check, c Compile either the compiler or libraries, using cargo check
+ clippy Run clippy (uses rustup/cargo-installed clippy binary)
+ fix Run cargo fix
+ fmt Run rustfmt
+ test, t Build and run some test suites
+ bench Build and run some benchmarks
+ doc, d Build documentation
+ clean Clean out build directories
+ dist Build distribution artifacts
+ install Install distribution artifacts
+ run, r Run tools contained in this repository
+ setup Create a config.toml (making it easier to use `x.py` itself)
+
+To learn more about a subcommand, run `./x.py <subcommand> -h`",
+ );
+
+ let mut opts = Options::new();
+ // Options common to all subcommands
+ opts.optflagmulti("v", "verbose", "use verbose output (-vv for very verbose)");
+ opts.optflag("i", "incremental", "use incremental compilation");
+ opts.optopt("", "config", "TOML configuration file for build", "FILE");
+ opts.optopt(
+ "",
+ "build-dir",
+ "Build directory, overrides `build.build-dir` in `config.toml`",
+ "DIR",
+ );
+ opts.optopt("", "build", "build target of the stage0 compiler", "BUILD");
+ opts.optmulti("", "host", "host targets to build", "HOST");
+ opts.optmulti("", "target", "target targets to build", "TARGET");
+ opts.optmulti("", "exclude", "build paths to exclude", "PATH");
+ opts.optflag(
+ "",
+ "include-default-paths",
+ "include default paths in addition to the provided ones",
+ );
+ opts.optopt("", "on-fail", "command to run on failure", "CMD");
+ opts.optflag("", "dry-run", "dry run; don't build anything");
+ opts.optopt(
+ "",
+ "stage",
+ "stage to build (indicates compiler to use/test, e.g., stage 0 uses the \
+ bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.)",
+ "N",
+ );
+ opts.optmulti(
+ "",
+ "keep-stage",
+ "stage(s) to keep without recompiling \
+ (pass multiple times to keep e.g., both stages 0 and 1)",
+ "N",
+ );
+ opts.optmulti(
+ "",
+ "keep-stage-std",
+ "stage(s) of the standard library to keep without recompiling \
+ (pass multiple times to keep e.g., both stages 0 and 1)",
+ "N",
+ );
+ opts.optopt("", "src", "path to the root of the rust checkout", "DIR");
+ let j_msg = format!(
+ "number of jobs to run in parallel; \
+ defaults to {} (this host's logical CPU count)",
+ num_cpus::get()
+ );
+ opts.optopt("j", "jobs", &j_msg, "JOBS");
+ opts.optflag("h", "help", "print this help message");
+ opts.optopt(
+ "",
+ "warnings",
+ "if value is deny, will deny warnings, otherwise use default",
+ "VALUE",
+ );
+ opts.optopt("", "error-format", "rustc error format", "FORMAT");
+ opts.optflag("", "json-output", "use message-format=json");
+ opts.optopt("", "color", "whether to use color in cargo and rustc output", "STYLE");
+ opts.optopt(
+ "",
+ "llvm-skip-rebuild",
+ "whether rebuilding llvm should be skipped \
+ a VALUE of TRUE indicates that llvm will not be rebuilt \
+ VALUE overrides the skip-rebuild option in config.toml.",
+ "VALUE",
+ );
+ opts.optopt(
+ "",
+ "rust-profile-generate",
+ "generate PGO profile with rustc build",
+ "PROFILE",
+ );
+ opts.optopt("", "rust-profile-use", "use PGO profile for rustc build", "PROFILE");
+ opts.optflag("", "llvm-profile-generate", "generate PGO profile with llvm built for rustc");
+ opts.optopt("", "llvm-profile-use", "use PGO profile for llvm build", "PROFILE");
+ opts.optmulti("A", "", "allow certain clippy lints", "OPT");
+ opts.optmulti("D", "", "deny certain clippy lints", "OPT");
+ opts.optmulti("W", "", "warn about certain clippy lints", "OPT");
+ opts.optmulti("F", "", "forbid certain clippy lints", "OPT");
+
+ // We can't use getopt to parse the options until we have completed specifying which
+ // options are valid, but under the current implementation, some options are conditional on
+ // the subcommand. Therefore we must manually identify the subcommand first, so that we can
+ // complete the definition of the options. Then we can use the getopt::Matches object from
+ // there on out.
+ let subcommand = match args.iter().find_map(|s| Kind::parse(&s)) {
+ Some(s) => s,
+ None => {
+ // No or an invalid subcommand -- show the general usage and subcommand help
+ // An exit code will be 0 when no subcommand is given, and 1 in case of an invalid
+ // subcommand.
+ println!("{}\n", subcommand_help);
+ let exit_code = if args.is_empty() { 0 } else { 1 };
+ crate::detail_exit(exit_code);
+ }
+ };
+
+ // Some subcommands get extra options
+ match subcommand {
+ Kind::Test => {
+ opts.optflag("", "no-fail-fast", "Run all tests regardless of failure");
+ opts.optmulti("", "skip", "skips tests matching SUBSTRING, if supported by test tool. May be passed multiple times", "SUBSTRING");
+ opts.optmulti(
+ "",
+ "test-args",
+ "extra arguments to be passed for the test tool being used \
+ (e.g. libtest, compiletest or rustdoc)",
+ "ARGS",
+ );
+ opts.optmulti(
+ "",
+ "rustc-args",
+ "extra options to pass the compiler when running tests",
+ "ARGS",
+ );
+ opts.optflag("", "no-doc", "do not run doc tests");
+ opts.optflag("", "doc", "only run doc tests");
+ opts.optflag("", "bless", "update all stderr/stdout files of failing ui tests");
+ opts.optflag("", "force-rerun", "rerun tests even if the inputs are unchanged");
+ opts.optopt(
+ "",
+ "compare-mode",
+ "mode describing what file the actual ui output will be compared to",
+ "COMPARE MODE",
+ );
+ opts.optopt(
+ "",
+ "pass",
+ "force {check,build,run}-pass tests to this mode.",
+ "check | build | run",
+ );
+ opts.optopt("", "run", "whether to execute run-* tests", "auto | always | never");
+ opts.optflag(
+ "",
+ "rustfix-coverage",
+ "enable this to generate a Rustfix coverage file, which is saved in \
+ `/<build_base>/rustfix_missing_coverage.txt`",
+ );
+ }
+ Kind::Check => {
+ opts.optflag("", "all-targets", "Check all targets");
+ }
+ Kind::Bench => {
+ opts.optmulti("", "test-args", "extra arguments", "ARGS");
+ }
+ Kind::Clippy => {
+ opts.optflag("", "fix", "automatically apply lint suggestions");
+ }
+ Kind::Doc => {
+ opts.optflag("", "open", "open the docs in a browser");
+ }
+ Kind::Clean => {
+ opts.optflag("", "all", "clean all build artifacts");
+ }
+ Kind::Format => {
+ opts.optflag("", "check", "check formatting instead of applying.");
+ }
+ _ => {}
+ };
+
+ // fn usage()
+ let usage = |exit_code: i32, opts: &Options, verbose: bool, subcommand_help: &str| -> ! {
+ let config = Config::parse(&["build".to_string()]);
+ let build = Build::new(config);
+ let paths = Builder::get_help(&build, subcommand);
+
+ println!("{}", opts.usage(subcommand_help));
+ if let Some(s) = paths {
+ if verbose {
+ println!("{}", s);
+ } else {
+ println!(
+ "Run `./x.py {} -h -v` to see a list of available paths.",
+ subcommand.as_str()
+ );
+ }
+ } else if verbose {
+ panic!("No paths available for subcommand `{}`", subcommand.as_str());
+ }
+ crate::detail_exit(exit_code);
+ };
+
+ // Done specifying what options are possible, so do the getopts parsing
+ let matches = opts.parse(args).unwrap_or_else(|e| {
+ // Invalid argument/option format
+ println!("\n{}\n", e);
+ usage(1, &opts, false, &subcommand_help);
+ });
+
+ // Extra sanity check to make sure we didn't hit this crazy corner case:
+ //
+ // ./x.py --frobulate clean build
+ // ^-- option ^ ^- actual subcommand
+ // \_ arg to option could be mistaken as subcommand
+ let mut pass_sanity_check = true;
+ match matches.free.get(0).and_then(|s| Kind::parse(&s)) {
+ Some(check_subcommand) => {
+ if check_subcommand != subcommand {
+ pass_sanity_check = false;
+ }
+ }
+ None => {
+ pass_sanity_check = false;
+ }
+ }
+ if !pass_sanity_check {
+ eprintln!("{}\n", subcommand_help);
+ eprintln!(
+ "Sorry, I couldn't figure out which subcommand you were trying to specify.\n\
+ You may need to move some options to after the subcommand.\n"
+ );
+ crate::detail_exit(1);
+ }
+ // Extra help text for some commands
+ match subcommand {
+ Kind::Build => {
+ subcommand_help.push_str(
+ "\n
+Arguments:
+ This subcommand accepts a number of paths to directories to the crates
+ and/or artifacts to compile. For example, for a quick build of a usable
+ compiler:
+
+ ./x.py build --stage 1 library/std
+
+ This will build a compiler and standard library from the local source code.
+ Once this is done, build/$ARCH/stage1 contains a usable compiler.
+
+ If no arguments are passed then the default artifacts for that stage are
+ compiled. For example:
+
+ ./x.py build --stage 0
+ ./x.py build ",
+ );
+ }
+ Kind::Check => {
+ subcommand_help.push_str(
+ "\n
+Arguments:
+ This subcommand accepts a number of paths to directories to the crates
+ and/or artifacts to compile. For example:
+
+ ./x.py check library/std
+
+ If no arguments are passed then many artifacts are checked.",
+ );
+ }
+ Kind::Clippy => {
+ subcommand_help.push_str(
+ "\n
+Arguments:
+ This subcommand accepts a number of paths to directories to the crates
+ and/or artifacts to run clippy against. For example:
+
+ ./x.py clippy library/core
+ ./x.py clippy library/core library/proc_macro",
+ );
+ }
+ Kind::Fix => {
+ subcommand_help.push_str(
+ "\n
+Arguments:
+ This subcommand accepts a number of paths to directories to the crates
+ and/or artifacts to run `cargo fix` against. For example:
+
+ ./x.py fix library/core
+ ./x.py fix library/core library/proc_macro",
+ );
+ }
+ Kind::Format => {
+ subcommand_help.push_str(
+ "\n
+Arguments:
+ This subcommand optionally accepts a `--check` flag which succeeds if formatting is correct and
+ fails if it is not. For example:
+
+ ./x.py fmt
+ ./x.py fmt --check",
+ );
+ }
+ Kind::Test => {
+ subcommand_help.push_str(
+ "\n
+Arguments:
+ This subcommand accepts a number of paths to test directories that
+ should be compiled and run. For example:
+
+ ./x.py test src/test/ui
+ ./x.py test library/std --test-args hash_map
+ ./x.py test library/std --stage 0 --no-doc
+ ./x.py test src/test/ui --bless
+ ./x.py test src/test/ui --compare-mode chalk
+
+ Note that `test src/test/* --stage N` does NOT depend on `build compiler/rustc --stage N`;
+ just like `build library/std --stage N` it tests the compiler produced by the previous
+ stage.
+
+ Execute tool tests with a tool name argument:
+
+ ./x.py test tidy
+
+ If no arguments are passed then the complete artifacts for that stage are
+ compiled and tested.
+
+ ./x.py test
+ ./x.py test --stage 1",
+ );
+ }
+ Kind::Doc => {
+ subcommand_help.push_str(
+ "\n
+Arguments:
+ This subcommand accepts a number of paths to directories of documentation
+ to build. For example:
+
+ ./x.py doc src/doc/book
+ ./x.py doc src/doc/nomicon
+ ./x.py doc src/doc/book library/std
+ ./x.py doc library/std --open
+
+ If no arguments are passed then everything is documented:
+
+ ./x.py doc
+ ./x.py doc --stage 1",
+ );
+ }
+ Kind::Run => {
+ subcommand_help.push_str(
+ "\n
+Arguments:
+ This subcommand accepts a number of paths to tools to build and run. For
+ example:
+
+ ./x.py run src/tools/expand-yaml-anchors
+
+ At least a tool needs to be called.",
+ );
+ }
+ Kind::Setup => {
+ subcommand_help.push_str(&format!(
+ "\n
+x.py setup creates a `config.toml` which changes the defaults for x.py itself.
+
+Arguments:
+ This subcommand accepts a 'profile' to use for builds. For example:
+
+ ./x.py setup library
+
+ The profile is optional and you will be prompted interactively if it is not given.
+ The following profiles are available:
+
+{}",
+ Profile::all_for_help(" ").trim_end()
+ ));
+ }
+ Kind::Bench | Kind::Clean | Kind::Dist | Kind::Install => {}
+ };
+ // Get any optional paths which occur after the subcommand
+ let mut paths = matches.free[1..].iter().map(|p| p.into()).collect::<Vec<PathBuf>>();
+
+ let verbose = matches.opt_present("verbose");
+
+ // User passed in -h/--help?
+ if matches.opt_present("help") {
+ usage(0, &opts, verbose, &subcommand_help);
+ }
+
+ let cmd = match subcommand {
+ Kind::Build => Subcommand::Build { paths },
+ Kind::Check => {
+ if matches.opt_present("all-targets") {
+ println!(
+ "Warning: --all-targets is now on by default and does not need to be passed explicitly."
+ );
+ }
+ Subcommand::Check { paths }
+ }
+ Kind::Clippy => Subcommand::Clippy {
+ paths,
+ fix: matches.opt_present("fix"),
+ clippy_lint_allow: matches.opt_strs("A"),
+ clippy_lint_warn: matches.opt_strs("W"),
+ clippy_lint_deny: matches.opt_strs("D"),
+ clippy_lint_forbid: matches.opt_strs("F"),
+ },
+ Kind::Fix => Subcommand::Fix { paths },
+ Kind::Test => Subcommand::Test {
+ paths,
+ bless: matches.opt_present("bless"),
+ force_rerun: matches.opt_present("force-rerun"),
+ compare_mode: matches.opt_str("compare-mode"),
+ pass: matches.opt_str("pass"),
+ run: matches.opt_str("run"),
+ skip: matches.opt_strs("skip"),
+ test_args: matches.opt_strs("test-args"),
+ rustc_args: matches.opt_strs("rustc-args"),
+ fail_fast: !matches.opt_present("no-fail-fast"),
+ rustfix_coverage: matches.opt_present("rustfix-coverage"),
+ doc_tests: if matches.opt_present("doc") {
+ DocTests::Only
+ } else if matches.opt_present("no-doc") {
+ DocTests::No
+ } else {
+ DocTests::Yes
+ },
+ },
+ Kind::Bench => Subcommand::Bench { paths, test_args: matches.opt_strs("test-args") },
+ Kind::Doc => Subcommand::Doc { paths, open: matches.opt_present("open") },
+ Kind::Clean => {
+ if !paths.is_empty() {
+ println!("\nclean does not take a path argument\n");
+ usage(1, &opts, verbose, &subcommand_help);
+ }
+
+ Subcommand::Clean { all: matches.opt_present("all") }
+ }
+ Kind::Format => Subcommand::Format { check: matches.opt_present("check"), paths },
+ Kind::Dist => Subcommand::Dist { paths },
+ Kind::Install => Subcommand::Install { paths },
+ Kind::Run => {
+ if paths.is_empty() {
+ println!("\nrun requires at least a path!\n");
+ usage(1, &opts, verbose, &subcommand_help);
+ }
+ Subcommand::Run { paths }
+ }
+ Kind::Setup => {
+ let profile = if paths.len() > 1 {
+ println!("\nat most one profile can be passed to setup\n");
+ usage(1, &opts, verbose, &subcommand_help)
+ } else if let Some(path) = paths.pop() {
+ let profile_string = t!(path.into_os_string().into_string().map_err(
+ |path| format!("{} is not a valid UTF8 string", path.to_string_lossy())
+ ));
+
+ profile_string.parse().unwrap_or_else(|err| {
+ eprintln!("error: {}", err);
+ eprintln!("help: the available profiles are:");
+ eprint!("{}", Profile::all_for_help("- "));
+ crate::detail_exit(1);
+ })
+ } else {
+ t!(crate::setup::interactive_path())
+ };
+ Subcommand::Setup { profile }
+ }
+ };
+
+ Flags {
+ verbose: matches.opt_count("verbose"),
+ stage: matches.opt_str("stage").map(|j| j.parse().expect("`stage` should be a number")),
+ dry_run: matches.opt_present("dry-run"),
+ on_fail: matches.opt_str("on-fail"),
+ rustc_error_format: matches.opt_str("error-format"),
+ json_output: matches.opt_present("json-output"),
+ keep_stage: matches
+ .opt_strs("keep-stage")
+ .into_iter()
+ .map(|j| j.parse().expect("`keep-stage` should be a number"))
+ .collect(),
+ keep_stage_std: matches
+ .opt_strs("keep-stage-std")
+ .into_iter()
+ .map(|j| j.parse().expect("`keep-stage-std` should be a number"))
+ .collect(),
+ host: if matches.opt_present("host") {
+ Some(
+ split(&matches.opt_strs("host"))
+ .into_iter()
+ .map(|x| TargetSelection::from_user(&x))
+ .collect::<Vec<_>>(),
+ )
+ } else {
+ None
+ },
+ target: if matches.opt_present("target") {
+ Some(
+ split(&matches.opt_strs("target"))
+ .into_iter()
+ .map(|x| TargetSelection::from_user(&x))
+ .collect::<Vec<_>>(),
+ )
+ } else {
+ None
+ },
+ config: matches.opt_str("config").map(PathBuf::from),
+ build_dir: matches.opt_str("build-dir").map(PathBuf::from),
+ jobs: matches.opt_str("jobs").map(|j| j.parse().expect("`jobs` should be a number")),
+ cmd,
+ incremental: matches.opt_present("incremental"),
+ exclude: split(&matches.opt_strs("exclude"))
+ .into_iter()
+ .map(|p| p.into())
+ .collect::<Vec<_>>(),
+ include_default_paths: matches.opt_present("include-default-paths"),
+ deny_warnings: parse_deny_warnings(&matches),
+ llvm_skip_rebuild: matches.opt_str("llvm-skip-rebuild").map(|s| s.to_lowercase()).map(
+ |s| s.parse::<bool>().expect("`llvm-skip-rebuild` should be either true or false"),
+ ),
+ color: matches
+ .opt_get_default("color", Color::Auto)
+ .expect("`color` should be `always`, `never`, or `auto`"),
+ rust_profile_use: matches.opt_str("rust-profile-use"),
+ rust_profile_generate: matches.opt_str("rust-profile-generate"),
+ llvm_profile_use: matches.opt_str("llvm-profile-use"),
+ llvm_profile_generate: matches.opt_present("llvm-profile-generate"),
+ }
+ }
+}
+
+impl Subcommand {
+ pub fn kind(&self) -> Kind {
+ match self {
+ Subcommand::Bench { .. } => Kind::Bench,
+ Subcommand::Build { .. } => Kind::Build,
+ Subcommand::Check { .. } => Kind::Check,
+ Subcommand::Clippy { .. } => Kind::Clippy,
+ Subcommand::Doc { .. } => Kind::Doc,
+ Subcommand::Fix { .. } => Kind::Fix,
+ Subcommand::Format { .. } => Kind::Format,
+ Subcommand::Test { .. } => Kind::Test,
+ Subcommand::Clean { .. } => Kind::Clean,
+ Subcommand::Dist { .. } => Kind::Dist,
+ Subcommand::Install { .. } => Kind::Install,
+ Subcommand::Run { .. } => Kind::Run,
+ Subcommand::Setup { .. } => Kind::Setup,
+ }
+ }
+
+ pub fn test_args(&self) -> Vec<&str> {
+ let mut args = vec![];
+
+ match *self {
+ Subcommand::Test { ref skip, .. } => {
+ for s in skip {
+ args.push("--skip");
+ args.push(s.as_str());
+ }
+ }
+ _ => (),
+ };
+
+ match *self {
+ Subcommand::Test { ref test_args, .. } | Subcommand::Bench { ref test_args, .. } => {
+ args.extend(test_args.iter().flat_map(|s| s.split_whitespace()))
+ }
+ _ => (),
+ }
+
+ args
+ }
+
+ pub fn rustc_args(&self) -> Vec<&str> {
+ match *self {
+ Subcommand::Test { ref rustc_args, .. } => {
+ rustc_args.iter().flat_map(|s| s.split_whitespace()).collect()
+ }
+ _ => Vec::new(),
+ }
+ }
+
+ pub fn fail_fast(&self) -> bool {
+ match *self {
+ Subcommand::Test { fail_fast, .. } => fail_fast,
+ _ => false,
+ }
+ }
+
+ pub fn doc_tests(&self) -> DocTests {
+ match *self {
+ Subcommand::Test { doc_tests, .. } => doc_tests,
+ _ => DocTests::Yes,
+ }
+ }
+
+ pub fn bless(&self) -> bool {
+ match *self {
+ Subcommand::Test { bless, .. } => bless,
+ _ => false,
+ }
+ }
+
+ pub fn force_rerun(&self) -> bool {
+ match *self {
+ Subcommand::Test { force_rerun, .. } => force_rerun,
+ _ => false,
+ }
+ }
+
+ pub fn rustfix_coverage(&self) -> bool {
+ match *self {
+ Subcommand::Test { rustfix_coverage, .. } => rustfix_coverage,
+ _ => false,
+ }
+ }
+
+ pub fn compare_mode(&self) -> Option<&str> {
+ match *self {
+ Subcommand::Test { ref compare_mode, .. } => compare_mode.as_ref().map(|s| &s[..]),
+ _ => None,
+ }
+ }
+
+ pub fn pass(&self) -> Option<&str> {
+ match *self {
+ Subcommand::Test { ref pass, .. } => pass.as_ref().map(|s| &s[..]),
+ _ => None,
+ }
+ }
+
+ pub fn run(&self) -> Option<&str> {
+ match *self {
+ Subcommand::Test { ref run, .. } => run.as_ref().map(|s| &s[..]),
+ _ => None,
+ }
+ }
+
+ pub fn open(&self) -> bool {
+ match *self {
+ Subcommand::Doc { open, .. } => open,
+ _ => false,
+ }
+ }
+}
+
+fn split(s: &[String]) -> Vec<String> {
+ s.iter().flat_map(|s| s.split(',')).filter(|s| !s.is_empty()).map(|s| s.to_string()).collect()
+}
+
+fn parse_deny_warnings(matches: &getopts::Matches) -> Option<bool> {
+ match matches.opt_str("warnings").as_deref() {
+ Some("deny") => Some(true),
+ Some("warn") => Some(false),
+ Some(value) => {
+ eprintln!(r#"invalid value for --warnings: {:?}, expected "warn" or "deny""#, value,);
+ crate::detail_exit(1);
+ }
+ None => None,
+ }
+}
diff --git a/src/bootstrap/format.rs b/src/bootstrap/format.rs
new file mode 100644
index 000000000..37322670e
--- /dev/null
+++ b/src/bootstrap/format.rs
@@ -0,0 +1,175 @@
+//! Runs rustfmt on the repository.
+
+use crate::builder::Builder;
+use crate::util::{output, t};
+use ignore::WalkBuilder;
+use std::collections::VecDeque;
+use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
+use std::sync::mpsc::SyncSender;
+
+fn rustfmt(src: &Path, rustfmt: &Path, paths: &[PathBuf], check: bool) -> impl FnMut() {
+ let mut cmd = Command::new(&rustfmt);
+ // avoid the submodule config paths from coming into play,
+ // we only allow a single global config for the workspace for now
+ cmd.arg("--config-path").arg(&src.canonicalize().unwrap());
+ cmd.arg("--edition").arg("2021");
+ cmd.arg("--unstable-features");
+ cmd.arg("--skip-children");
+ if check {
+ cmd.arg("--check");
+ }
+ cmd.args(paths);
+ let cmd_debug = format!("{:?}", cmd);
+ let mut cmd = cmd.spawn().expect("running rustfmt");
+ // poor man's async: return a closure that'll wait for rustfmt's completion
+ move || {
+ let status = cmd.wait().unwrap();
+ if !status.success() {
+ eprintln!(
+ "Running `{}` failed.\nIf you're running `tidy`, \
+ try again with `--bless`. Or, if you just want to format \
+ code, run `./x.py fmt` instead.",
+ cmd_debug,
+ );
+ crate::detail_exit(1);
+ }
+ }
+}
+
+#[derive(serde::Deserialize)]
+struct RustfmtConfig {
+ ignore: Vec<String>,
+}
+
+pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) {
+ if build.config.dry_run {
+ return;
+ }
+ let mut builder = ignore::types::TypesBuilder::new();
+ builder.add_defaults();
+ builder.select("rust");
+ let matcher = builder.build().unwrap();
+ let rustfmt_config = build.src.join("rustfmt.toml");
+ if !rustfmt_config.exists() {
+ eprintln!("Not running formatting checks; rustfmt.toml does not exist.");
+ eprintln!("This may happen in distributed tarballs.");
+ return;
+ }
+ let rustfmt_config = t!(std::fs::read_to_string(&rustfmt_config));
+ let rustfmt_config: RustfmtConfig = t!(toml::from_str(&rustfmt_config));
+ let mut ignore_fmt = ignore::overrides::OverrideBuilder::new(&build.src);
+ for ignore in rustfmt_config.ignore {
+ ignore_fmt.add(&format!("!{}", ignore)).expect(&ignore);
+ }
+ let git_available = match Command::new("git")
+ .arg("--version")
+ .stdout(Stdio::null())
+ .stderr(Stdio::null())
+ .status()
+ {
+ Ok(status) => status.success(),
+ Err(_) => false,
+ };
+ if git_available {
+ let in_working_tree = match build
+ .config
+ .git()
+ .arg("rev-parse")
+ .arg("--is-inside-work-tree")
+ .stdout(Stdio::null())
+ .stderr(Stdio::null())
+ .status()
+ {
+ Ok(status) => status.success(),
+ Err(_) => false,
+ };
+ if in_working_tree {
+ let untracked_paths_output = output(
+ build.config.git().arg("status").arg("--porcelain").arg("--untracked-files=normal"),
+ );
+ let untracked_paths = untracked_paths_output
+ .lines()
+ .filter(|entry| entry.starts_with("??"))
+ .map(|entry| {
+ entry.split(' ').nth(1).expect("every git status entry should list a path")
+ });
+ for untracked_path in untracked_paths {
+ println!("skip untracked path {} during rustfmt invocations", untracked_path);
+ // The leading `/` makes it an exact match against the
+ // repository root, rather than a glob. Without that, if you
+ // have `foo.rs` in the repository root it will also match
+ // against anything like `compiler/rustc_foo/src/foo.rs`,
+ // preventing the latter from being formatted.
+ ignore_fmt.add(&format!("!/{}", untracked_path)).expect(&untracked_path);
+ }
+ } else {
+ println!("Not in git tree. Skipping git-aware format checks");
+ }
+ } else {
+ println!("Could not find usable git. Skipping git-aware format checks");
+ }
+ let ignore_fmt = ignore_fmt.build().unwrap();
+
+ let rustfmt_path = build.initial_rustfmt().unwrap_or_else(|| {
+ eprintln!("./x.py fmt is not supported on this channel");
+ crate::detail_exit(1);
+ });
+ assert!(rustfmt_path.exists(), "{}", rustfmt_path.display());
+ let src = build.src.clone();
+ let (tx, rx): (SyncSender<PathBuf>, _) = std::sync::mpsc::sync_channel(128);
+ let walker = match paths.get(0) {
+ Some(first) => {
+ let mut walker = WalkBuilder::new(first);
+ for path in &paths[1..] {
+ walker.add(path);
+ }
+ walker
+ }
+ None => WalkBuilder::new(src.clone()),
+ }
+ .types(matcher)
+ .overrides(ignore_fmt)
+ .build_parallel();
+
+ // there is a lot of blocking involved in spawning a child process and reading files to format.
+ // spawn more processes than available concurrency to keep the CPU busy
+ let max_processes = build.jobs() as usize * 2;
+
+ // spawn child processes on a separate thread so we can batch entries we have received from ignore
+ let thread = std::thread::spawn(move || {
+ let mut children = VecDeque::new();
+ while let Ok(path) = rx.recv() {
+ // try getting a few more paths from the channel to amortize the overhead of spawning processes
+ let paths: Vec<_> = rx.try_iter().take(7).chain(std::iter::once(path)).collect();
+
+ let child = rustfmt(&src, &rustfmt_path, paths.as_slice(), check);
+ children.push_back(child);
+
+ if children.len() >= max_processes {
+ // await oldest child
+ children.pop_front().unwrap()();
+ }
+ }
+
+ // await remaining children
+ for mut child in children {
+ child();
+ }
+ });
+
+ walker.run(|| {
+ let tx = tx.clone();
+ Box::new(move |entry| {
+ let entry = t!(entry);
+ if entry.file_type().map_or(false, |t| t.is_file()) {
+ t!(tx.send(entry.into_path()));
+ }
+ ignore::WalkState::Continue
+ })
+ });
+
+ drop(tx);
+
+ thread.join().unwrap();
+}
diff --git a/src/bootstrap/install.rs b/src/bootstrap/install.rs
new file mode 100644
index 000000000..6e49f39ff
--- /dev/null
+++ b/src/bootstrap/install.rs
@@ -0,0 +1,292 @@
+//! Implementation of the install aspects of the compiler.
+//!
+//! This module is responsible for installing the standard library,
+//! compiler, and documentation.
+
+use std::env;
+use std::fs;
+use std::path::{Component, Path, PathBuf};
+use std::process::Command;
+
+use crate::util::t;
+
+use crate::dist;
+use crate::tarball::GeneratedTarball;
+use crate::Compiler;
+
+use crate::builder::{Builder, RunConfig, ShouldRun, Step};
+use crate::config::{Config, TargetSelection};
+
+#[cfg(target_os = "illumos")]
+const SHELL: &str = "bash";
+#[cfg(not(target_os = "illumos"))]
+const SHELL: &str = "sh";
+
+// We have to run a few shell scripts, which choke quite a bit on both `\`
+// characters and on `C:\` paths, so normalize both of them away.
+fn sanitize_sh(path: &Path) -> String {
+ let path = path.to_str().unwrap().replace("\\", "/");
+ return change_drive(unc_to_lfs(&path)).unwrap_or(path);
+
+ fn unc_to_lfs(s: &str) -> &str {
+ s.strip_prefix("//?/").unwrap_or(s)
+ }
+
+ fn change_drive(s: &str) -> Option<String> {
+ let mut ch = s.chars();
+ let drive = ch.next().unwrap_or('C');
+ if ch.next() != Some(':') {
+ return None;
+ }
+ if ch.next() != Some('/') {
+ return None;
+ }
+ Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
+ }
+}
+
+fn install_sh(
+ builder: &Builder<'_>,
+ package: &str,
+ stage: u32,
+ host: Option<TargetSelection>,
+ tarball: &GeneratedTarball,
+) {
+ builder.info(&format!("Install {} stage{} ({:?})", package, stage, host));
+
+ let prefix = default_path(&builder.config.prefix, "/usr/local");
+ let sysconfdir = prefix.join(default_path(&builder.config.sysconfdir, "/etc"));
+ let datadir = prefix.join(default_path(&builder.config.datadir, "share"));
+ let docdir = prefix.join(default_path(&builder.config.docdir, "share/doc/rust"));
+ let mandir = prefix.join(default_path(&builder.config.mandir, "share/man"));
+ let libdir = prefix.join(default_path(&builder.config.libdir, "lib"));
+ let bindir = prefix.join(&builder.config.bindir); // Default in config.rs
+
+ let empty_dir = builder.out.join("tmp/empty_dir");
+ t!(fs::create_dir_all(&empty_dir));
+
+ let mut cmd = Command::new(SHELL);
+ cmd.current_dir(&empty_dir)
+ .arg(sanitize_sh(&tarball.decompressed_output().join("install.sh")))
+ .arg(format!("--prefix={}", prepare_dir(prefix)))
+ .arg(format!("--sysconfdir={}", prepare_dir(sysconfdir)))
+ .arg(format!("--datadir={}", prepare_dir(datadir)))
+ .arg(format!("--docdir={}", prepare_dir(docdir)))
+ .arg(format!("--bindir={}", prepare_dir(bindir)))
+ .arg(format!("--libdir={}", prepare_dir(libdir)))
+ .arg(format!("--mandir={}", prepare_dir(mandir)))
+ .arg("--disable-ldconfig");
+ builder.run(&mut cmd);
+ t!(fs::remove_dir_all(&empty_dir));
+}
+
+fn default_path(config: &Option<PathBuf>, default: &str) -> PathBuf {
+ config.as_ref().cloned().unwrap_or_else(|| PathBuf::from(default))
+}
+
+fn prepare_dir(mut path: PathBuf) -> String {
+ // The DESTDIR environment variable is a standard way to install software in a subdirectory
+ // while keeping the original directory structure, even if the prefix or other directories
+ // contain absolute paths.
+ //
+ // More information on the environment variable is available here:
+ // https://www.gnu.org/prep/standards/html_node/DESTDIR.html
+ if let Some(destdir) = env::var_os("DESTDIR").map(PathBuf::from) {
+ let without_destdir = path.clone();
+ path = destdir;
+ // Custom .join() which ignores disk roots.
+ for part in without_destdir.components() {
+ if let Component::Normal(s) = part {
+ path.push(s)
+ }
+ }
+ }
+
+ // The installation command is not executed from the current directory, but from a temporary
+ // directory. To prevent relative paths from breaking this converts relative paths to absolute
+ // paths. std::fs::canonicalize is not used as that requires the path to actually be present.
+ if path.is_relative() {
+ path = std::env::current_dir().expect("failed to get the current directory").join(path);
+ assert!(path.is_absolute(), "could not make the path relative");
+ }
+
+ sanitize_sh(&path)
+}
+
+macro_rules! install {
+ (($sel:ident, $builder:ident, $_config:ident),
+ $($name:ident,
+ $condition_name: ident = $path_or_alias: literal,
+ $default_cond:expr,
+ only_hosts: $only_hosts:expr,
+ $run_item:block $(, $c:ident)*;)+) => {
+ $(
+ #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+ pub struct $name {
+ pub compiler: Compiler,
+ pub target: TargetSelection,
+ }
+
+ impl $name {
+ #[allow(dead_code)]
+ fn should_build(config: &Config) -> bool {
+ config.extended && config.tools.as_ref()
+ .map_or(true, |t| t.contains($path_or_alias))
+ }
+ }
+
+ impl Step for $name {
+ type Output = ();
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = $only_hosts;
+ $(const $c: bool = true;)*
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let $_config = &run.builder.config;
+ run.$condition_name($path_or_alias).default_condition($default_cond)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure($name {
+ compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
+ target: run.target,
+ });
+ }
+
+ fn run($sel, $builder: &Builder<'_>) {
+ $run_item
+ }
+ })+
+ }
+}
+
+install!((self, builder, _config),
+ Docs, path = "src/doc", _config.docs, only_hosts: false, {
+ let tarball = builder.ensure(dist::Docs { host: self.target }).expect("missing docs");
+ install_sh(builder, "docs", self.compiler.stage, Some(self.target), &tarball);
+ };
+ Std, path = "library/std", true, only_hosts: false, {
+ for target in &builder.targets {
+ // `expect` should be safe, only None when host != build, but this
+ // only runs when host == build
+ let tarball = builder.ensure(dist::Std {
+ compiler: self.compiler,
+ target: *target
+ }).expect("missing std");
+ install_sh(builder, "std", self.compiler.stage, Some(*target), &tarball);
+ }
+ };
+ Cargo, alias = "cargo", Self::should_build(_config), only_hosts: true, {
+ let tarball = builder
+ .ensure(dist::Cargo { compiler: self.compiler, target: self.target })
+ .expect("missing cargo");
+ install_sh(builder, "cargo", self.compiler.stage, Some(self.target), &tarball);
+ };
+ Rls, alias = "rls", Self::should_build(_config), only_hosts: true, {
+ if let Some(tarball) = builder.ensure(dist::Rls { compiler: self.compiler, target: self.target }) {
+ install_sh(builder, "rls", self.compiler.stage, Some(self.target), &tarball);
+ } else {
+ builder.info(
+ &format!("skipping Install RLS stage{} ({})", self.compiler.stage, self.target),
+ );
+ }
+ };
+ RustAnalyzer, alias = "rust-analyzer", Self::should_build(_config), only_hosts: true, {
+ if let Some(tarball) =
+ builder.ensure(dist::RustAnalyzer { compiler: self.compiler, target: self.target })
+ {
+ install_sh(builder, "rust-analyzer", self.compiler.stage, Some(self.target), &tarball);
+ } else {
+ builder.info(
+ &format!("skipping Install rust-analyzer stage{} ({})", self.compiler.stage, self.target),
+ );
+ }
+ };
+ Clippy, alias = "clippy", Self::should_build(_config), only_hosts: true, {
+ let tarball = builder
+ .ensure(dist::Clippy { compiler: self.compiler, target: self.target })
+ .expect("missing clippy");
+ install_sh(builder, "clippy", self.compiler.stage, Some(self.target), &tarball);
+ };
+ Miri, alias = "miri", Self::should_build(_config), only_hosts: true, {
+ if let Some(tarball) = builder.ensure(dist::Miri { compiler: self.compiler, target: self.target }) {
+ install_sh(builder, "miri", self.compiler.stage, Some(self.target), &tarball);
+ } else {
+ builder.info(
+ &format!("skipping Install miri stage{} ({})", self.compiler.stage, self.target),
+ );
+ }
+ };
+ Rustfmt, alias = "rustfmt", Self::should_build(_config), only_hosts: true, {
+ if let Some(tarball) = builder.ensure(dist::Rustfmt {
+ compiler: self.compiler,
+ target: self.target
+ }) {
+ install_sh(builder, "rustfmt", self.compiler.stage, Some(self.target), &tarball);
+ } else {
+ builder.info(
+ &format!("skipping Install Rustfmt stage{} ({})", self.compiler.stage, self.target),
+ );
+ }
+ };
+ RustDemangler, alias = "rust-demangler", Self::should_build(_config), only_hosts: true, {
+ // Note: Even though `should_build` may return true for `extended` default tools,
+ // dist::RustDemangler may still return None, unless the target-dependent `profiler` config
+ // is also true, or the `tools` array explicitly includes "rust-demangler".
+ if let Some(tarball) = builder.ensure(dist::RustDemangler {
+ compiler: self.compiler,
+ target: self.target
+ }) {
+ install_sh(builder, "rust-demangler", self.compiler.stage, Some(self.target), &tarball);
+ } else {
+ builder.info(
+ &format!("skipping Install RustDemangler stage{} ({})",
+ self.compiler.stage, self.target),
+ );
+ }
+ };
+ Analysis, alias = "analysis", Self::should_build(_config), only_hosts: false, {
+ // `expect` should be safe, only None with host != build, but this
+ // only uses the `build` compiler
+ let tarball = builder.ensure(dist::Analysis {
+ // Find the actual compiler (handling the full bootstrap option) which
+ // produced the save-analysis data because that data isn't copied
+ // through the sysroot uplifting.
+ compiler: builder.compiler_for(builder.top_stage, builder.config.build, self.target),
+ target: self.target
+ }).expect("missing analysis");
+ install_sh(builder, "analysis", self.compiler.stage, Some(self.target), &tarball);
+ };
+ Rustc, path = "compiler/rustc", true, only_hosts: true, {
+ let tarball = builder.ensure(dist::Rustc {
+ compiler: builder.compiler(builder.top_stage, self.target),
+ });
+ install_sh(builder, "rustc", self.compiler.stage, Some(self.target), &tarball);
+ };
+);
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Src {
+ pub stage: u32,
+}
+
+impl Step for Src {
+ type Output = ();
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let config = &run.builder.config;
+ let cond = config.extended && config.tools.as_ref().map_or(true, |t| t.contains("src"));
+ run.path("src").default_condition(cond)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Src { stage: run.builder.top_stage });
+ }
+
+ fn run(self, builder: &Builder<'_>) {
+ let tarball = builder.ensure(dist::Src);
+ install_sh(builder, "src", self.stage, None, &tarball);
+ }
+}
diff --git a/src/bootstrap/job.rs b/src/bootstrap/job.rs
new file mode 100644
index 000000000..5c0322e18
--- /dev/null
+++ b/src/bootstrap/job.rs
@@ -0,0 +1,140 @@
+//! Job management on Windows for bootstrapping
+//!
+//! Most of the time when you're running a build system (e.g., make) you expect
+//! Ctrl-C or abnormal termination to actually terminate the entire tree of
+//! process in play, not just the one at the top. This currently works "by
+//! default" on Unix platforms because Ctrl-C actually sends a signal to the
+//! *process group* rather than the parent process, so everything will get torn
+//! down. On Windows, however, this does not happen and Ctrl-C just kills the
+//! parent process.
+//!
+//! To achieve the same semantics on Windows we use Job Objects to ensure that
+//! all processes die at the same time. Job objects have a mode of operation
+//! where when all handles to the object are closed it causes all child
+//! processes associated with the object to be terminated immediately.
+//! Conveniently whenever a process in the job object spawns a new process the
+//! child will be associated with the job object as well. This means if we add
+//! ourselves to the job object we create then everything will get torn down!
+//!
+//! Unfortunately most of the time the build system is actually called from a
+//! python wrapper (which manages things like building the build system) so this
+//! all doesn't quite cut it so far. To go the last mile we duplicate the job
+//! object handle into our parent process (a python process probably) and then
+//! close our own handle. This means that the only handle to the job object
+//! resides in the parent python process, so when python dies the whole build
+//! system dies (as one would probably expect!).
+//!
+//! Note that this module has a #[cfg(windows)] above it as none of this logic
+//! is required on Unix.
+
+#![allow(nonstandard_style, dead_code)]
+
+use crate::Build;
+use std::env;
+use std::io;
+use std::mem;
+use std::ptr;
+
+use winapi::shared::minwindef::{DWORD, FALSE, LPVOID};
+use winapi::um::errhandlingapi::SetErrorMode;
+use winapi::um::handleapi::{CloseHandle, DuplicateHandle};
+use winapi::um::jobapi2::{AssignProcessToJobObject, CreateJobObjectW, SetInformationJobObject};
+use winapi::um::processthreadsapi::{GetCurrentProcess, OpenProcess};
+use winapi::um::winbase::{BELOW_NORMAL_PRIORITY_CLASS, SEM_NOGPFAULTERRORBOX};
+use winapi::um::winnt::{
+ JobObjectExtendedLimitInformation, DUPLICATE_SAME_ACCESS, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,
+ JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOB_OBJECT_LIMIT_PRIORITY_CLASS, PROCESS_DUP_HANDLE,
+};
+
+pub unsafe fn setup(build: &mut Build) {
+ // Enable the Windows Error Reporting dialog which msys disables,
+ // so we can JIT debug rustc
+ let mode = SetErrorMode(0);
+ SetErrorMode(mode & !SEM_NOGPFAULTERRORBOX);
+
+ // Create a new job object for us to use
+ let job = CreateJobObjectW(ptr::null_mut(), ptr::null());
+ assert!(!job.is_null(), "{}", io::Error::last_os_error());
+
+ // Indicate that when all handles to the job object are gone that all
+ // process in the object should be killed. Note that this includes our
+ // entire process tree by default because we've added ourselves and our
+ // children will reside in the job by default.
+ let mut info = mem::zeroed::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>();
+ info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
+ if build.config.low_priority {
+ info.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_PRIORITY_CLASS;
+ info.BasicLimitInformation.PriorityClass = BELOW_NORMAL_PRIORITY_CLASS;
+ }
+ let r = SetInformationJobObject(
+ job,
+ JobObjectExtendedLimitInformation,
+ &mut info as *mut _ as LPVOID,
+ mem::size_of_val(&info) as DWORD,
+ );
+ assert!(r != 0, "{}", io::Error::last_os_error());
+
+ // Assign our process to this job object. Note that if this fails, one very
+ // likely reason is that we are ourselves already in a job object! This can
+ // happen on the build bots that we've got for Windows, or if just anyone
+ // else is instrumenting the build. In this case we just bail out
+ // immediately and assume that they take care of it.
+ //
+ // Also note that nested jobs (why this might fail) are supported in recent
+ // versions of Windows, but the version of Windows that our bots are running
+ // at least don't support nested job objects.
+ let r = AssignProcessToJobObject(job, GetCurrentProcess());
+ if r == 0 {
+ CloseHandle(job);
+ return;
+ }
+
+ // If we've got a parent process (e.g., the python script that called us)
+ // then move ownership of this job object up to them. That way if the python
+ // script is killed (e.g., via ctrl-c) then we'll all be torn down.
+ //
+ // If we don't have a parent (e.g., this was run directly) then we
+ // intentionally leak the job object handle. When our process exits
+ // (normally or abnormally) it will close the handle implicitly, causing all
+ // processes in the job to be cleaned up.
+ let pid = match env::var("BOOTSTRAP_PARENT_ID") {
+ Ok(s) => s,
+ Err(..) => return,
+ };
+
+ let parent = OpenProcess(PROCESS_DUP_HANDLE, FALSE, pid.parse().unwrap());
+
+ // If we get a null parent pointer here, it is possible that either
+ // we have got an invalid pid or the parent process has been closed.
+ // Since the first case rarely happens
+ // (only when wrongly setting the environmental variable),
+ // so it might be better to improve the experience of the second case
+ // when users have interrupted the parent process and we don't finish
+ // duplicating the handle yet.
+ // We just need close the job object if that occurs.
+ if parent.is_null() {
+ CloseHandle(job);
+ return;
+ }
+
+ let mut parent_handle = ptr::null_mut();
+ let r = DuplicateHandle(
+ GetCurrentProcess(),
+ job,
+ parent,
+ &mut parent_handle,
+ 0,
+ FALSE,
+ DUPLICATE_SAME_ACCESS,
+ );
+
+ // If this failed, well at least we tried! An example of DuplicateHandle
+ // failing in the past has been when the wrong python2 package spawned this
+ // build system (e.g., the `python2` package in MSYS instead of
+ // `mingw-w64-x86_64-python2`. Not sure why it failed, but the "failure
+ // mode" here is that we only clean everything up when the build system
+ // dies, not when the python parent does, so not too bad.
+ if r != 0 {
+ CloseHandle(job);
+ }
+}
diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs
new file mode 100644
index 000000000..d265277b4
--- /dev/null
+++ b/src/bootstrap/lib.rs
@@ -0,0 +1,1679 @@
+//! Implementation of rustbuild, the Rust build system.
+//!
+//! This module, and its descendants, are the implementation of the Rust build
+//! system. Most of this build system is backed by Cargo but the outer layer
+//! here serves as the ability to orchestrate calling Cargo, sequencing Cargo
+//! builds, building artifacts like LLVM, etc. The goals of rustbuild are:
+//!
+//! * To be an easily understandable, easily extensible, and maintainable build
+//! system.
+//! * Leverage standard tools in the Rust ecosystem to build the compiler, aka
+//! crates.io and Cargo.
+//! * A standard interface to build across all platforms, including MSVC
+//!
+//! ## Architecture
+//!
+//! The build system defers most of the complicated logic managing invocations
+//! of rustc and rustdoc to Cargo itself. However, moving through various stages
+//! and copying artifacts is still necessary for it to do. Each time rustbuild
+//! is invoked, it will iterate through the list of predefined steps and execute
+//! each serially in turn if it matches the paths passed or is a default rule.
+//! For each step rustbuild relies on the step internally being incremental and
+//! parallel. Note, though, that the `-j` parameter to rustbuild gets forwarded
+//! to appropriate test harnesses and such.
+//!
+//! Most of the "meaty" steps that matter are backed by Cargo, which does indeed
+//! have its own parallelism and incremental management. Later steps, like
+//! tests, aren't incremental and simply run the entire suite currently.
+//! However, compiletest itself tries to avoid running tests when the artifacts
+//! that are involved (mainly the compiler) haven't changed.
+//!
+//! When you execute `x.py build`, the steps executed are:
+//!
+//! * First, the python script is run. This will automatically download the
+//! stage0 rustc and cargo according to `src/stage0.json`, or use the cached
+//! versions if they're available. These are then used to compile rustbuild
+//! itself (using Cargo). Finally, control is then transferred to rustbuild.
+//!
+//! * Rustbuild takes over, performs sanity checks, probes the environment,
+//! reads configuration, and starts executing steps as it reads the command
+//! line arguments (paths) or going through the default rules.
+//!
+//! The build output will be something like the following:
+//!
+//! Building stage0 std artifacts
+//! Copying stage0 std
+//! Building stage0 test artifacts
+//! Copying stage0 test
+//! Building stage0 compiler artifacts
+//! Copying stage0 rustc
+//! Assembling stage1 compiler
+//! Building stage1 std artifacts
+//! Copying stage1 std
+//! Building stage1 test artifacts
+//! Copying stage1 test
+//! Building stage1 compiler artifacts
+//! Copying stage1 rustc
+//! Assembling stage2 compiler
+//! Uplifting stage1 std
+//! Uplifting stage1 test
+//! Uplifting stage1 rustc
+//!
+//! Let's disect that a little:
+//!
+//! ## Building stage0 {std,test,compiler} artifacts
+//!
+//! These steps use the provided (downloaded, usually) compiler to compile the
+//! local Rust source into libraries we can use.
+//!
+//! ## Copying stage0 {std,test,rustc}
+//!
+//! This copies the build output from Cargo into
+//! `build/$HOST/stage0-sysroot/lib/rustlib/$ARCH/lib`. FIXME: this step's
+//! documentation should be expanded -- the information already here may be
+//! incorrect.
+//!
+//! ## Assembling stage1 compiler
+//!
+//! This copies the libraries we built in "building stage0 ... artifacts" into
+//! the stage1 compiler's lib directory. These are the host libraries that the
+//! compiler itself uses to run. These aren't actually used by artifacts the new
+//! compiler generates. This step also copies the rustc and rustdoc binaries we
+//! generated into build/$HOST/stage/bin.
+//!
+//! The stage1/bin/rustc is a fully functional compiler, but it doesn't yet have
+//! any libraries to link built binaries or libraries to. The next 3 steps will
+//! provide those libraries for it; they are mostly equivalent to constructing
+//! the stage1/bin compiler so we don't go through them individually.
+//!
+//! ## Uplifting stage1 {std,test,rustc}
+//!
+//! This step copies the libraries from the stage1 compiler sysroot into the
+//! stage2 compiler. This is done to avoid rebuilding the compiler; libraries
+//! we'd build in this step should be identical (in function, if not necessarily
+//! identical on disk) so there's no need to recompile the compiler again. Note
+//! that if you want to, you can enable the full-bootstrap option to change this
+//! behavior.
+//!
+//! Each step is driven by a separate Cargo project and rustbuild orchestrates
+//! copying files between steps and otherwise preparing for Cargo to run.
+//!
+//! ## Further information
+//!
+//! More documentation can be found in each respective module below, and you can
+//! also check out the `src/bootstrap/README.md` file for more information.
+
+use std::cell::{Cell, RefCell};
+use std::collections::{HashMap, HashSet};
+use std::env;
+use std::fs::{self, File};
+use std::io;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+use std::str;
+
+use filetime::FileTime;
+use once_cell::sync::OnceCell;
+
+use crate::builder::Kind;
+use crate::config::{LlvmLibunwind, TargetSelection};
+use crate::util::{
+ check_run, exe, libdir, mtime, output, run, run_suppressed, try_run, try_run_suppressed, CiEnv,
+};
+
+mod builder;
+mod cache;
+mod cc_detect;
+mod channel;
+mod check;
+mod clean;
+mod compile;
+mod config;
+mod dist;
+mod doc;
+mod flags;
+mod format;
+mod install;
+mod metadata;
+mod native;
+mod run;
+mod sanity;
+mod setup;
+mod tarball;
+mod test;
+mod tool;
+mod toolstate;
+pub mod util;
+
+#[cfg(feature = "build-metrics")]
+mod metrics;
+
+#[cfg(windows)]
+mod job;
+
+#[cfg(all(unix, not(target_os = "haiku")))]
+mod job {
+ pub unsafe fn setup(build: &mut crate::Build) {
+ if build.config.low_priority {
+ libc::setpriority(libc::PRIO_PGRP as _, 0, 10);
+ }
+ }
+}
+
+#[cfg(any(target_os = "haiku", target_os = "hermit", not(any(unix, windows))))]
+mod job {
+ pub unsafe fn setup(_build: &mut crate::Build) {}
+}
+
+pub use crate::builder::PathSet;
+use crate::cache::{Interned, INTERNER};
+pub use crate::config::Config;
+pub use crate::flags::Subcommand;
+
+const LLVM_TOOLS: &[&str] = &[
+ "llvm-cov", // used to generate coverage report
+ "llvm-nm", // used to inspect binaries; it shows symbol names, their sizes and visibility
+ "llvm-objcopy", // used to transform ELFs into binary format which flashing tools consume
+ "llvm-objdump", // used to disassemble programs
+ "llvm-profdata", // used to inspect and merge files generated by profiles
+ "llvm-readobj", // used to get information from ELFs/objects that the other tools don't provide
+ "llvm-size", // used to prints the size of the linker sections of a program
+ "llvm-strip", // used to discard symbols from binary files to reduce their size
+ "llvm-ar", // used for creating and modifying archive files
+ "llvm-as", // used to convert LLVM assembly to LLVM bitcode
+ "llvm-dis", // used to disassemble LLVM bitcode
+ "llc", // used to compile LLVM bytecode
+ "opt", // used to optimize LLVM bytecode
+];
+
+pub const VERSION: usize = 2;
+
+/// Extra --check-cfg to add when building
+/// (Mode restriction, config name, config values (if any))
+const EXTRA_CHECK_CFGS: &[(Option<Mode>, &'static str, Option<&[&'static str]>)] = &[
+ (None, "bootstrap", None),
+ (Some(Mode::Rustc), "parallel_compiler", None),
+ (Some(Mode::ToolRustc), "parallel_compiler", None),
+ (Some(Mode::Std), "stdarch_intel_sde", None),
+ (Some(Mode::Std), "no_fp_fmt_parse", None),
+ (Some(Mode::Std), "no_global_oom_handling", None),
+ (Some(Mode::Std), "freebsd12", None),
+ (Some(Mode::Std), "backtrace_in_libstd", None),
+ /* Extra values not defined in the built-in targets yet, but used in std */
+ (Some(Mode::Std), "target_env", Some(&["libnx"])),
+ (Some(Mode::Std), "target_os", Some(&["watchos"])),
+ (
+ Some(Mode::Std),
+ "target_arch",
+ Some(&["asmjs", "spirv", "nvptx", "nvptx64", "le32", "xtensa"]),
+ ),
+ /* Extra names used by dependencies */
+ // FIXME: Used by rustfmt is their test but is invalid (neither cargo nor bootstrap ever set
+ // this config) should probably by removed or use a allow attribute.
+ (Some(Mode::ToolRustc), "release", None),
+ // FIXME: Used by stdarch in their test, should use a allow attribute instead.
+ (Some(Mode::Std), "dont_compile_me", None),
+ // FIXME: Used by serde_json, but we should not be triggering on external dependencies.
+ (Some(Mode::Rustc), "no_btreemap_remove_entry", None),
+ (Some(Mode::ToolRustc), "no_btreemap_remove_entry", None),
+ // FIXME: Used by crossbeam-utils, but we should not be triggering on external dependencies.
+ (Some(Mode::Rustc), "crossbeam_loom", None),
+ (Some(Mode::ToolRustc), "crossbeam_loom", None),
+ // FIXME: Used by proc-macro2, but we should not be triggering on external dependencies.
+ (Some(Mode::Rustc), "span_locations", None),
+ (Some(Mode::ToolRustc), "span_locations", None),
+];
+
+/// A structure representing a Rust compiler.
+///
+/// Each compiler has a `stage` that it is associated with and a `host` that
+/// corresponds to the platform the compiler runs on. This structure is used as
+/// a parameter to many methods below.
+#[derive(Eq, PartialOrd, Ord, PartialEq, Clone, Copy, Hash, Debug)]
+pub struct Compiler {
+ stage: u32,
+ host: TargetSelection,
+}
+
+#[derive(PartialEq, Eq, Copy, Clone, Debug)]
+pub enum DocTests {
+ /// Run normal tests and doc tests (default).
+ Yes,
+ /// Do not run any doc tests.
+ No,
+ /// Only run doc tests.
+ Only,
+}
+
+pub enum GitRepo {
+ Rustc,
+ Llvm,
+}
+
+/// Global configuration for the build system.
+///
+/// This structure transitively contains all configuration for the build system.
+/// All filesystem-encoded configuration is in `config`, all flags are in
+/// `flags`, and then parsed or probed information is listed in the keys below.
+///
+/// This structure is a parameter of almost all methods in the build system,
+/// although most functions are implemented as free functions rather than
+/// methods specifically on this structure itself (to make it easier to
+/// organize).
+pub struct Build {
+ /// User-specified configuration from `config.toml`.
+ config: Config,
+
+ // Version information
+ version: String,
+
+ // Properties derived from the above configuration
+ src: PathBuf,
+ out: PathBuf,
+ bootstrap_out: PathBuf,
+ rust_info: channel::GitInfo,
+ cargo_info: channel::GitInfo,
+ rls_info: channel::GitInfo,
+ rust_analyzer_info: channel::GitInfo,
+ clippy_info: channel::GitInfo,
+ miri_info: channel::GitInfo,
+ rustfmt_info: channel::GitInfo,
+ in_tree_llvm_info: channel::GitInfo,
+ local_rebuild: bool,
+ fail_fast: bool,
+ doc_tests: DocTests,
+ verbosity: usize,
+
+ // Targets for which to build
+ build: TargetSelection,
+ hosts: Vec<TargetSelection>,
+ targets: Vec<TargetSelection>,
+
+ initial_rustc: PathBuf,
+ initial_cargo: PathBuf,
+ initial_lld: PathBuf,
+ initial_libdir: PathBuf,
+
+ // Runtime state filled in later on
+ // C/C++ compilers and archiver for all targets
+ cc: HashMap<TargetSelection, cc::Tool>,
+ cxx: HashMap<TargetSelection, cc::Tool>,
+ ar: HashMap<TargetSelection, PathBuf>,
+ ranlib: HashMap<TargetSelection, PathBuf>,
+ // Miscellaneous
+ // allow bidirectional lookups: both name -> path and path -> name
+ crates: HashMap<Interned<String>, Crate>,
+ crate_paths: HashMap<PathBuf, Interned<String>>,
+ is_sudo: bool,
+ ci_env: CiEnv,
+ delayed_failures: RefCell<Vec<String>>,
+ prerelease_version: Cell<Option<u32>>,
+ tool_artifacts:
+ RefCell<HashMap<TargetSelection, HashMap<String, (&'static str, PathBuf, Vec<String>)>>>,
+
+ #[cfg(feature = "build-metrics")]
+ metrics: metrics::BuildMetrics,
+}
+
+#[derive(Debug)]
+struct Crate {
+ name: Interned<String>,
+ deps: HashSet<Interned<String>>,
+ path: PathBuf,
+}
+
+impl Crate {
+ fn local_path(&self, build: &Build) -> PathBuf {
+ self.path.strip_prefix(&build.config.src).unwrap().into()
+ }
+}
+
+/// When building Rust various objects are handled differently.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub enum DependencyType {
+ /// Libraries originating from proc-macros.
+ Host,
+ /// Typical Rust libraries.
+ Target,
+ /// Non Rust libraries and objects shipped to ease usage of certain targets.
+ TargetSelfContained,
+}
+
+/// The various "modes" of invoking Cargo.
+///
+/// These entries currently correspond to the various output directories of the
+/// build system, with each mod generating output in a different directory.
+#[derive(Debug, Hash, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub enum Mode {
+ /// Build the standard library, placing output in the "stageN-std" directory.
+ Std,
+
+ /// Build librustc, and compiler libraries, placing output in the "stageN-rustc" directory.
+ Rustc,
+
+ /// Build a codegen backend for rustc, placing the output in the "stageN-codegen" directory.
+ Codegen,
+
+ /// Build a tool, placing output in the "stage0-bootstrap-tools"
+ /// directory. This is for miscellaneous sets of tools that are built
+ /// using the bootstrap stage0 compiler in its entirety (target libraries
+ /// and all). Typically these tools compile with stable Rust.
+ ToolBootstrap,
+
+ /// Build a tool which uses the locally built std, placing output in the
+ /// "stageN-tools" directory. Its usage is quite rare, mainly used by
+ /// compiletest which needs libtest.
+ ToolStd,
+
+ /// Build a tool which uses the locally built rustc and the target std,
+ /// placing the output in the "stageN-tools" directory. This is used for
+ /// anything that needs a fully functional rustc, such as rustdoc, clippy,
+ /// cargo, rls, rustfmt, miri, etc.
+ ToolRustc,
+}
+
+impl Mode {
+ pub fn is_tool(&self) -> bool {
+ matches!(self, Mode::ToolBootstrap | Mode::ToolRustc | Mode::ToolStd)
+ }
+
+ pub fn must_support_dlopen(&self) -> bool {
+ matches!(self, Mode::Std | Mode::Codegen)
+ }
+}
+
+pub enum CLang {
+ C,
+ Cxx,
+}
+
+impl Build {
+ /// Creates a new set of build configuration from the `flags` on the command
+ /// line and the filesystem `config`.
+ ///
+ /// By default all build output will be placed in the current directory.
+ pub fn new(config: Config) -> Build {
+ let src = config.src.clone();
+ let out = config.out.clone();
+
+ #[cfg(unix)]
+ // keep this consistent with the equivalent check in x.py:
+ // https://github.com/rust-lang/rust/blob/a8a33cf27166d3eabaffc58ed3799e054af3b0c6/src/bootstrap/bootstrap.py#L796-L797
+ let is_sudo = match env::var_os("SUDO_USER") {
+ Some(_sudo_user) => {
+ let uid = unsafe { libc::getuid() };
+ uid == 0
+ }
+ None => false,
+ };
+ #[cfg(not(unix))]
+ let is_sudo = false;
+
+ let ignore_git = config.ignore_git;
+ let rust_info = channel::GitInfo::new(ignore_git, &src);
+ let cargo_info = channel::GitInfo::new(ignore_git, &src.join("src/tools/cargo"));
+ let rls_info = channel::GitInfo::new(ignore_git, &src.join("src/tools/rls"));
+ let rust_analyzer_info =
+ channel::GitInfo::new(ignore_git, &src.join("src/tools/rust-analyzer"));
+ let clippy_info = channel::GitInfo::new(ignore_git, &src.join("src/tools/clippy"));
+ let miri_info = channel::GitInfo::new(ignore_git, &src.join("src/tools/miri"));
+ let rustfmt_info = channel::GitInfo::new(ignore_git, &src.join("src/tools/rustfmt"));
+
+ // we always try to use git for LLVM builds
+ let in_tree_llvm_info = channel::GitInfo::new(false, &src.join("src/llvm-project"));
+
+ let initial_target_libdir_str = if config.dry_run {
+ "/dummy/lib/path/to/lib/".to_string()
+ } else {
+ output(
+ Command::new(&config.initial_rustc)
+ .arg("--target")
+ .arg(config.build.rustc_target_arg())
+ .arg("--print")
+ .arg("target-libdir"),
+ )
+ };
+ let initial_target_dir = Path::new(&initial_target_libdir_str).parent().unwrap();
+ let initial_lld = initial_target_dir.join("bin").join("rust-lld");
+
+ let initial_sysroot = if config.dry_run {
+ "/dummy".to_string()
+ } else {
+ output(Command::new(&config.initial_rustc).arg("--print").arg("sysroot"))
+ };
+ let initial_libdir = initial_target_dir
+ .parent()
+ .unwrap()
+ .parent()
+ .unwrap()
+ .strip_prefix(initial_sysroot.trim())
+ .unwrap()
+ .to_path_buf();
+
+ let version = std::fs::read_to_string(src.join("src").join("version"))
+ .expect("failed to read src/version");
+ let version = version.trim();
+
+ let bootstrap_out = if std::env::var("BOOTSTRAP_PYTHON").is_ok() {
+ out.join("bootstrap").join("debug")
+ } else {
+ let workspace_target_dir = std::env::var("CARGO_TARGET_DIR")
+ .map(PathBuf::from)
+ .unwrap_or_else(|_| src.join("target"));
+ let bootstrap_out = workspace_target_dir.join("debug");
+ if !bootstrap_out.join("rustc").exists() && !cfg!(test) {
+ // this restriction can be lifted whenever https://github.com/rust-lang/rfcs/pull/3028 is implemented
+ panic!("run `cargo build --bins` before `cargo run`")
+ }
+ bootstrap_out
+ };
+
+ let mut build = Build {
+ initial_rustc: config.initial_rustc.clone(),
+ initial_cargo: config.initial_cargo.clone(),
+ initial_lld,
+ initial_libdir,
+ local_rebuild: config.local_rebuild,
+ fail_fast: config.cmd.fail_fast(),
+ doc_tests: config.cmd.doc_tests(),
+ verbosity: config.verbose,
+
+ build: config.build,
+ hosts: config.hosts.clone(),
+ targets: config.targets.clone(),
+
+ config,
+ version: version.to_string(),
+ src,
+ out,
+ bootstrap_out,
+
+ rust_info,
+ cargo_info,
+ rls_info,
+ rust_analyzer_info,
+ clippy_info,
+ miri_info,
+ rustfmt_info,
+ in_tree_llvm_info,
+ cc: HashMap::new(),
+ cxx: HashMap::new(),
+ ar: HashMap::new(),
+ ranlib: HashMap::new(),
+ crates: HashMap::new(),
+ crate_paths: HashMap::new(),
+ is_sudo,
+ ci_env: CiEnv::current(),
+ delayed_failures: RefCell::new(Vec::new()),
+ prerelease_version: Cell::new(None),
+ tool_artifacts: Default::default(),
+
+ #[cfg(feature = "build-metrics")]
+ metrics: metrics::BuildMetrics::init(),
+ };
+
+ build.verbose("finding compilers");
+ cc_detect::find(&mut build);
+ // When running `setup`, the profile is about to change, so any requirements we have now may
+ // be different on the next invocation. Don't check for them until the next time x.py is
+ // run. This is ok because `setup` never runs any build commands, so it won't fail if commands are missing.
+ if !matches!(build.config.cmd, Subcommand::Setup { .. }) {
+ build.verbose("running sanity check");
+ sanity::check(&mut build);
+ }
+
+ // If local-rust is the same major.minor as the current version, then force a
+ // local-rebuild
+ let local_version_verbose =
+ output(Command::new(&build.initial_rustc).arg("--version").arg("--verbose"));
+ let local_release = local_version_verbose
+ .lines()
+ .filter_map(|x| x.strip_prefix("release:"))
+ .next()
+ .unwrap()
+ .trim();
+ if local_release.split('.').take(2).eq(version.split('.').take(2)) {
+ build.verbose(&format!("auto-detected local-rebuild {}", local_release));
+ build.local_rebuild = true;
+ }
+
+ // Make sure we update these before gathering metadata so we don't get an error about missing
+ // Cargo.toml files.
+ let rust_submodules = [
+ "src/tools/rust-installer",
+ "src/tools/cargo",
+ "src/tools/rls",
+ "src/tools/miri",
+ "library/backtrace",
+ "library/stdarch",
+ ];
+ for s in rust_submodules {
+ build.update_submodule(Path::new(s));
+ }
+
+ build.verbose("learning about cargo");
+ metadata::build(&mut build);
+
+ build
+ }
+
+ // modified from `check_submodule` and `update_submodule` in bootstrap.py
+ /// Given a path to the directory of a submodule, update it.
+ ///
+ /// `relative_path` should be relative to the root of the git repository, not an absolute path.
+ pub(crate) fn update_submodule(&self, relative_path: &Path) {
+ fn dir_is_empty(dir: &Path) -> bool {
+ t!(std::fs::read_dir(dir)).next().is_none()
+ }
+
+ if !self.config.submodules(&self.rust_info) {
+ return;
+ }
+
+ let absolute_path = self.config.src.join(relative_path);
+
+ // NOTE: The check for the empty directory is here because when running x.py the first time,
+ // the submodule won't be checked out. Check it out now so we can build it.
+ if !channel::GitInfo::new(false, &absolute_path).is_git() && !dir_is_empty(&absolute_path) {
+ return;
+ }
+
+ // check_submodule
+ let checked_out_hash =
+ output(Command::new("git").args(&["rev-parse", "HEAD"]).current_dir(&absolute_path));
+ // update_submodules
+ let recorded = output(
+ Command::new("git")
+ .args(&["ls-tree", "HEAD"])
+ .arg(relative_path)
+ .current_dir(&self.config.src),
+ );
+ let actual_hash = recorded
+ .split_whitespace()
+ .nth(2)
+ .unwrap_or_else(|| panic!("unexpected output `{}`", recorded));
+
+ // update_submodule
+ if actual_hash == checked_out_hash.trim_end() {
+ // already checked out
+ return;
+ }
+
+ println!("Updating submodule {}", relative_path.display());
+ self.run(
+ Command::new("git")
+ .args(&["submodule", "-q", "sync"])
+ .arg(relative_path)
+ .current_dir(&self.config.src),
+ );
+
+ // Try passing `--progress` to start, then run git again without if that fails.
+ let update = |progress: bool| {
+ let mut git = Command::new("git");
+ git.args(&["submodule", "update", "--init", "--recursive", "--depth=1"]);
+ if progress {
+ git.arg("--progress");
+ }
+ git.arg(relative_path).current_dir(&self.config.src);
+ git
+ };
+ // NOTE: doesn't use `try_run` because this shouldn't print an error if it fails.
+ if !update(true).status().map_or(false, |status| status.success()) {
+ self.run(&mut update(false));
+ }
+
+ self.run(Command::new("git").args(&["reset", "-q", "--hard"]).current_dir(&absolute_path));
+ self.run(Command::new("git").args(&["clean", "-qdfx"]).current_dir(absolute_path));
+ }
+
+ /// If any submodule has been initialized already, sync it unconditionally.
+ /// This avoids contributors checking in a submodule change by accident.
+ pub fn maybe_update_submodules(&self) {
+ // Avoid running git when there isn't a git checkout.
+ if !self.config.submodules(&self.rust_info) {
+ return;
+ }
+ let output = output(
+ self.config
+ .git()
+ .args(&["config", "--file"])
+ .arg(&self.config.src.join(".gitmodules"))
+ .args(&["--get-regexp", "path"]),
+ );
+ for line in output.lines() {
+ // Look for `submodule.$name.path = $path`
+ // Sample output: `submodule.src/rust-installer.path src/tools/rust-installer`
+ let submodule = Path::new(line.splitn(2, ' ').nth(1).unwrap());
+ // Don't update the submodule unless it's already been cloned.
+ if channel::GitInfo::new(false, submodule).is_git() {
+ self.update_submodule(submodule);
+ }
+ }
+ }
+
+ /// Executes the entire build, as configured by the flags and configuration.
+ pub fn build(&mut self) {
+ unsafe {
+ job::setup(self);
+ }
+
+ self.maybe_update_submodules();
+
+ if let Subcommand::Format { check, paths } = &self.config.cmd {
+ return format::format(&builder::Builder::new(&self), *check, &paths);
+ }
+
+ if let Subcommand::Clean { all } = self.config.cmd {
+ return clean::clean(self, all);
+ }
+
+ if let Subcommand::Setup { profile } = &self.config.cmd {
+ return setup::setup(&self.config, *profile);
+ }
+
+ {
+ let builder = builder::Builder::new(&self);
+ if let Some(path) = builder.paths.get(0) {
+ if path == Path::new("nonexistent/path/to/trigger/cargo/metadata") {
+ return;
+ }
+ }
+ }
+
+ if !self.config.dry_run {
+ {
+ self.config.dry_run = true;
+ let builder = builder::Builder::new(&self);
+ builder.execute_cli();
+ }
+ self.config.dry_run = false;
+ let builder = builder::Builder::new(&self);
+ builder.execute_cli();
+ } else {
+ let builder = builder::Builder::new(&self);
+ builder.execute_cli();
+ }
+
+ // Check for postponed failures from `test --no-fail-fast`.
+ let failures = self.delayed_failures.borrow();
+ if failures.len() > 0 {
+ eprintln!("\n{} command(s) did not execute successfully:\n", failures.len());
+ for failure in failures.iter() {
+ eprintln!(" - {}\n", failure);
+ }
+ detail_exit(1);
+ }
+
+ #[cfg(feature = "build-metrics")]
+ self.metrics.persist(self);
+ }
+
+ /// Clear out `dir` if `input` is newer.
+ ///
+ /// After this executes, it will also ensure that `dir` exists.
+ fn clear_if_dirty(&self, dir: &Path, input: &Path) -> bool {
+ let stamp = dir.join(".stamp");
+ let mut cleared = false;
+ if mtime(&stamp) < mtime(input) {
+ self.verbose(&format!("Dirty - {}", dir.display()));
+ let _ = fs::remove_dir_all(dir);
+ cleared = true;
+ } else if stamp.exists() {
+ return cleared;
+ }
+ t!(fs::create_dir_all(dir));
+ t!(File::create(stamp));
+ cleared
+ }
+
+ /// Gets the space-separated set of activated features for the standard
+ /// library.
+ fn std_features(&self, target: TargetSelection) -> String {
+ let mut features = "panic-unwind".to_string();
+
+ match self.config.llvm_libunwind(target) {
+ LlvmLibunwind::InTree => features.push_str(" llvm-libunwind"),
+ LlvmLibunwind::System => features.push_str(" system-llvm-libunwind"),
+ LlvmLibunwind::No => {}
+ }
+ if self.config.backtrace {
+ features.push_str(" backtrace");
+ }
+ if self.config.profiler_enabled(target) {
+ features.push_str(" profiler");
+ }
+ features
+ }
+
+ /// Gets the space-separated set of activated features for the compiler.
+ fn rustc_features(&self, kind: Kind) -> String {
+ let mut features = vec![];
+ if self.config.jemalloc {
+ features.push("jemalloc");
+ }
+ if self.config.llvm_enabled() || kind == Kind::Check {
+ features.push("llvm");
+ }
+ // keep in sync with `bootstrap/compile.rs:rustc_cargo_env`
+ if self.config.rustc_parallel {
+ features.push("rustc_use_parallel_compiler");
+ }
+
+ // If debug logging is on, then we want the default for tracing:
+ // https://github.com/tokio-rs/tracing/blob/3dd5c03d907afdf2c39444a29931833335171554/tracing/src/level_filters.rs#L26
+ // which is everything (including debug/trace/etc.)
+ // if its unset, if debug_assertions is on, then debug_logging will also be on
+ // as well as tracing *ignoring* this feature when debug_assertions is on
+ if !self.config.rust_debug_logging {
+ features.push("max_level_info");
+ }
+
+ features.join(" ")
+ }
+
+ /// Component directory that Cargo will produce output into (e.g.
+ /// release/debug)
+ fn cargo_dir(&self) -> &'static str {
+ if self.config.rust_optimize { "release" } else { "debug" }
+ }
+
+ fn tools_dir(&self, compiler: Compiler) -> PathBuf {
+ let out = self
+ .out
+ .join(&*compiler.host.triple)
+ .join(format!("stage{}-tools-bin", compiler.stage));
+ t!(fs::create_dir_all(&out));
+ out
+ }
+
+ /// Returns the root directory for all output generated in a particular
+ /// stage when running with a particular host compiler.
+ ///
+ /// The mode indicates what the root directory is for.
+ fn stage_out(&self, compiler: Compiler, mode: Mode) -> PathBuf {
+ let suffix = match mode {
+ Mode::Std => "-std",
+ Mode::Rustc => "-rustc",
+ Mode::Codegen => "-codegen",
+ Mode::ToolBootstrap => "-bootstrap-tools",
+ Mode::ToolStd | Mode::ToolRustc => "-tools",
+ };
+ self.out.join(&*compiler.host.triple).join(format!("stage{}{}", compiler.stage, suffix))
+ }
+
+ /// Returns the root output directory for all Cargo output in a given stage,
+ /// running a particular compiler, whether or not we're building the
+ /// standard library, and targeting the specified architecture.
+ fn cargo_out(&self, compiler: Compiler, mode: Mode, target: TargetSelection) -> PathBuf {
+ self.stage_out(compiler, mode).join(&*target.triple).join(self.cargo_dir())
+ }
+
+ /// Root output directory for LLVM compiled for `target`
+ ///
+ /// Note that if LLVM is configured externally then the directory returned
+ /// will likely be empty.
+ fn llvm_out(&self, target: TargetSelection) -> PathBuf {
+ self.out.join(&*target.triple).join("llvm")
+ }
+
+ fn lld_out(&self, target: TargetSelection) -> PathBuf {
+ self.out.join(&*target.triple).join("lld")
+ }
+
+ /// Output directory for all documentation for a target
+ fn doc_out(&self, target: TargetSelection) -> PathBuf {
+ self.out.join(&*target.triple).join("doc")
+ }
+
+ fn test_out(&self, target: TargetSelection) -> PathBuf {
+ self.out.join(&*target.triple).join("test")
+ }
+
+ /// Output directory for all documentation for a target
+ fn compiler_doc_out(&self, target: TargetSelection) -> PathBuf {
+ self.out.join(&*target.triple).join("compiler-doc")
+ }
+
+ /// Output directory for some generated md crate documentation for a target (temporary)
+ fn md_doc_out(&self, target: TargetSelection) -> Interned<PathBuf> {
+ INTERNER.intern_path(self.out.join(&*target.triple).join("md-doc"))
+ }
+
+ /// Returns `true` if no custom `llvm-config` is set for the specified target.
+ ///
+ /// If no custom `llvm-config` was specified then Rust's llvm will be used.
+ fn is_rust_llvm(&self, target: TargetSelection) -> bool {
+ if self.config.llvm_from_ci && target == self.config.build {
+ return true;
+ }
+
+ match self.config.target_config.get(&target) {
+ Some(ref c) => c.llvm_config.is_none(),
+ None => true,
+ }
+ }
+
+ /// Returns the path to `FileCheck` binary for the specified target
+ fn llvm_filecheck(&self, target: TargetSelection) -> PathBuf {
+ let target_config = self.config.target_config.get(&target);
+ if let Some(s) = target_config.and_then(|c| c.llvm_filecheck.as_ref()) {
+ s.to_path_buf()
+ } else if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) {
+ let llvm_bindir = output(Command::new(s).arg("--bindir"));
+ let filecheck = Path::new(llvm_bindir.trim()).join(exe("FileCheck", target));
+ if filecheck.exists() {
+ filecheck
+ } else {
+ // On Fedora the system LLVM installs FileCheck in the
+ // llvm subdirectory of the libdir.
+ let llvm_libdir = output(Command::new(s).arg("--libdir"));
+ let lib_filecheck =
+ Path::new(llvm_libdir.trim()).join("llvm").join(exe("FileCheck", target));
+ if lib_filecheck.exists() {
+ lib_filecheck
+ } else {
+ // Return the most normal file name, even though
+ // it doesn't exist, so that any error message
+ // refers to that.
+ filecheck
+ }
+ }
+ } else {
+ let base = self.llvm_out(target).join("build");
+ let base = if !self.ninja() && target.contains("msvc") {
+ if self.config.llvm_optimize {
+ if self.config.llvm_release_debuginfo {
+ base.join("RelWithDebInfo")
+ } else {
+ base.join("Release")
+ }
+ } else {
+ base.join("Debug")
+ }
+ } else {
+ base
+ };
+ base.join("bin").join(exe("FileCheck", target))
+ }
+ }
+
+ /// Directory for libraries built from C/C++ code and shared between stages.
+ fn native_dir(&self, target: TargetSelection) -> PathBuf {
+ self.out.join(&*target.triple).join("native")
+ }
+
+ /// Root output directory for rust_test_helpers library compiled for
+ /// `target`
+ fn test_helpers_out(&self, target: TargetSelection) -> PathBuf {
+ self.native_dir(target).join("rust-test-helpers")
+ }
+
+ /// Adds the `RUST_TEST_THREADS` env var if necessary
+ fn add_rust_test_threads(&self, cmd: &mut Command) {
+ if env::var_os("RUST_TEST_THREADS").is_none() {
+ cmd.env("RUST_TEST_THREADS", self.jobs().to_string());
+ }
+ }
+
+ /// Returns the libdir of the snapshot compiler.
+ fn rustc_snapshot_libdir(&self) -> PathBuf {
+ self.rustc_snapshot_sysroot().join(libdir(self.config.build))
+ }
+
+ /// Returns the sysroot of the snapshot compiler.
+ fn rustc_snapshot_sysroot(&self) -> &Path {
+ static SYSROOT_CACHE: OnceCell<PathBuf> = once_cell::sync::OnceCell::new();
+ SYSROOT_CACHE.get_or_init(|| {
+ let mut rustc = Command::new(&self.initial_rustc);
+ rustc.args(&["--print", "sysroot"]);
+ output(&mut rustc).trim().into()
+ })
+ }
+
+ /// Runs a command, printing out nice contextual information if it fails.
+ fn run(&self, cmd: &mut Command) {
+ if self.config.dry_run {
+ return;
+ }
+ self.verbose(&format!("running: {:?}", cmd));
+ run(cmd, self.is_verbose())
+ }
+
+ /// Runs a command, printing out nice contextual information if it fails.
+ fn run_quiet(&self, cmd: &mut Command) {
+ if self.config.dry_run {
+ return;
+ }
+ self.verbose(&format!("running: {:?}", cmd));
+ run_suppressed(cmd)
+ }
+
+ /// Runs a command, printing out nice contextual information if it fails.
+ /// Exits if the command failed to execute at all, otherwise returns its
+ /// `status.success()`.
+ fn try_run(&self, cmd: &mut Command) -> bool {
+ if self.config.dry_run {
+ return true;
+ }
+ self.verbose(&format!("running: {:?}", cmd));
+ try_run(cmd, self.is_verbose())
+ }
+
+ /// Runs a command, printing out nice contextual information if it fails.
+ /// Exits if the command failed to execute at all, otherwise returns its
+ /// `status.success()`.
+ fn try_run_quiet(&self, cmd: &mut Command) -> bool {
+ if self.config.dry_run {
+ return true;
+ }
+ self.verbose(&format!("running: {:?}", cmd));
+ try_run_suppressed(cmd)
+ }
+
+ /// Runs a command, printing out nice contextual information if it fails.
+ /// Returns false if do not execute at all, otherwise returns its
+ /// `status.success()`.
+ fn check_run(&self, cmd: &mut Command) -> bool {
+ if self.config.dry_run {
+ return true;
+ }
+ self.verbose(&format!("running: {:?}", cmd));
+ check_run(cmd, self.is_verbose())
+ }
+
+ pub fn is_verbose(&self) -> bool {
+ self.verbosity > 0
+ }
+
+ /// Prints a message if this build is configured in verbose mode.
+ fn verbose(&self, msg: &str) {
+ if self.is_verbose() {
+ println!("{}", msg);
+ }
+ }
+
+ pub fn is_verbose_than(&self, level: usize) -> bool {
+ self.verbosity > level
+ }
+
+ /// Prints a message if this build is configured in more verbose mode than `level`.
+ fn verbose_than(&self, level: usize, msg: &str) {
+ if self.is_verbose_than(level) {
+ println!("{}", msg);
+ }
+ }
+
+ fn info(&self, msg: &str) {
+ if self.config.dry_run {
+ return;
+ }
+ println!("{}", msg);
+ }
+
+ /// Returns the number of parallel jobs that have been configured for this
+ /// build.
+ fn jobs(&self) -> u32 {
+ self.config.jobs.unwrap_or_else(|| num_cpus::get() as u32)
+ }
+
+ fn debuginfo_map_to(&self, which: GitRepo) -> Option<String> {
+ if !self.config.rust_remap_debuginfo {
+ return None;
+ }
+
+ match which {
+ GitRepo::Rustc => {
+ let sha = self.rust_sha().unwrap_or(&self.version);
+ Some(format!("/rustc/{}", sha))
+ }
+ GitRepo::Llvm => Some(String::from("/rustc/llvm")),
+ }
+ }
+
+ /// Returns the path to the C compiler for the target specified.
+ fn cc(&self, target: TargetSelection) -> &Path {
+ self.cc[&target].path()
+ }
+
+ /// Returns a list of flags to pass to the C compiler for the target
+ /// specified.
+ fn cflags(&self, target: TargetSelection, which: GitRepo, c: CLang) -> Vec<String> {
+ let base = match c {
+ CLang::C => &self.cc[&target],
+ CLang::Cxx => &self.cxx[&target],
+ };
+
+ // Filter out -O and /O (the optimization flags) that we picked up from
+ // cc-rs because the build scripts will determine that for themselves.
+ let mut base = base
+ .args()
+ .iter()
+ .map(|s| s.to_string_lossy().into_owned())
+ .filter(|s| !s.starts_with("-O") && !s.starts_with("/O"))
+ .collect::<Vec<String>>();
+
+ // If we're compiling on macOS then we add a few unconditional flags
+ // indicating that we want libc++ (more filled out than libstdc++) and
+ // we want to compile for 10.7. This way we can ensure that
+ // LLVM/etc are all properly compiled.
+ if target.contains("apple-darwin") {
+ base.push("-stdlib=libc++".into());
+ }
+
+ // Work around an apparently bad MinGW / GCC optimization,
+ // See: https://lists.llvm.org/pipermail/cfe-dev/2016-December/051980.html
+ // See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=78936
+ if &*target.triple == "i686-pc-windows-gnu" {
+ base.push("-fno-omit-frame-pointer".into());
+ }
+
+ if let Some(map_to) = self.debuginfo_map_to(which) {
+ let map = format!("{}={}", self.src.display(), map_to);
+ let cc = self.cc(target);
+ if cc.ends_with("clang") || cc.ends_with("gcc") {
+ base.push(format!("-fdebug-prefix-map={}", map));
+ } else if cc.ends_with("clang-cl.exe") {
+ base.push("-Xclang".into());
+ base.push(format!("-fdebug-prefix-map={}", map));
+ }
+ }
+ base
+ }
+
+ /// Returns the path to the `ar` archive utility for the target specified.
+ fn ar(&self, target: TargetSelection) -> Option<&Path> {
+ self.ar.get(&target).map(|p| &**p)
+ }
+
+ /// Returns the path to the `ranlib` utility for the target specified.
+ fn ranlib(&self, target: TargetSelection) -> Option<&Path> {
+ self.ranlib.get(&target).map(|p| &**p)
+ }
+
+ /// Returns the path to the C++ compiler for the target specified.
+ fn cxx(&self, target: TargetSelection) -> Result<&Path, String> {
+ match self.cxx.get(&target) {
+ Some(p) => Ok(p.path()),
+ None => {
+ Err(format!("target `{}` is not configured as a host, only as a target", target))
+ }
+ }
+ }
+
+ /// Returns the path to the linker for the given target if it needs to be overridden.
+ fn linker(&self, target: TargetSelection) -> Option<&Path> {
+ if let Some(linker) = self.config.target_config.get(&target).and_then(|c| c.linker.as_ref())
+ {
+ Some(linker)
+ } else if target.contains("vxworks") {
+ // need to use CXX compiler as linker to resolve the exception functions
+ // that are only existed in CXX libraries
+ Some(self.cxx[&target].path())
+ } else if target != self.config.build
+ && util::use_host_linker(target)
+ && !target.contains("msvc")
+ {
+ Some(self.cc(target))
+ } else if self.config.use_lld && !self.is_fuse_ld_lld(target) && self.build == target {
+ Some(&self.initial_lld)
+ } else {
+ None
+ }
+ }
+
+ // LLD is used through `-fuse-ld=lld` rather than directly.
+ // Only MSVC targets use LLD directly at the moment.
+ fn is_fuse_ld_lld(&self, target: TargetSelection) -> bool {
+ self.config.use_lld && !target.contains("msvc")
+ }
+
+ fn lld_flags(&self, target: TargetSelection) -> impl Iterator<Item = String> {
+ let mut options = [None, None];
+
+ if self.config.use_lld {
+ if self.is_fuse_ld_lld(target) {
+ options[0] = Some("-Clink-arg=-fuse-ld=lld".to_string());
+ }
+
+ let threads = if target.contains("windows") { "/threads:1" } else { "--threads=1" };
+ options[1] = Some(format!("-Clink-arg=-Wl,{}", threads));
+ }
+
+ IntoIterator::into_iter(options).flatten()
+ }
+
+ /// Returns if this target should statically link the C runtime, if specified
+ fn crt_static(&self, target: TargetSelection) -> Option<bool> {
+ if target.contains("pc-windows-msvc") {
+ Some(true)
+ } else {
+ self.config.target_config.get(&target).and_then(|t| t.crt_static)
+ }
+ }
+
+ /// Returns the "musl root" for this `target`, if defined
+ fn musl_root(&self, target: TargetSelection) -> Option<&Path> {
+ self.config
+ .target_config
+ .get(&target)
+ .and_then(|t| t.musl_root.as_ref())
+ .or_else(|| self.config.musl_root.as_ref())
+ .map(|p| &**p)
+ }
+
+ /// Returns the "musl libdir" for this `target`.
+ fn musl_libdir(&self, target: TargetSelection) -> Option<PathBuf> {
+ let t = self.config.target_config.get(&target)?;
+ if let libdir @ Some(_) = &t.musl_libdir {
+ return libdir.clone();
+ }
+ self.musl_root(target).map(|root| root.join("lib"))
+ }
+
+ /// Returns the sysroot for the wasi target, if defined
+ fn wasi_root(&self, target: TargetSelection) -> Option<&Path> {
+ self.config.target_config.get(&target).and_then(|t| t.wasi_root.as_ref()).map(|p| &**p)
+ }
+
+ /// Returns `true` if this is a no-std `target`, if defined
+ fn no_std(&self, target: TargetSelection) -> Option<bool> {
+ self.config.target_config.get(&target).map(|t| t.no_std)
+ }
+
+ /// Returns `true` if the target will be tested using the `remote-test-client`
+ /// and `remote-test-server` binaries.
+ fn remote_tested(&self, target: TargetSelection) -> bool {
+ self.qemu_rootfs(target).is_some()
+ || target.contains("android")
+ || env::var_os("TEST_DEVICE_ADDR").is_some()
+ }
+
+ /// Returns the root of the "rootfs" image that this target will be using,
+ /// if one was configured.
+ ///
+ /// If `Some` is returned then that means that tests for this target are
+ /// emulated with QEMU and binaries will need to be shipped to the emulator.
+ fn qemu_rootfs(&self, target: TargetSelection) -> Option<&Path> {
+ self.config.target_config.get(&target).and_then(|t| t.qemu_rootfs.as_ref()).map(|p| &**p)
+ }
+
+ /// Path to the python interpreter to use
+ fn python(&self) -> &Path {
+ if self.config.build.ends_with("apple-darwin") {
+ // Force /usr/bin/python3 on macOS for LLDB tests because we're loading the
+ // LLDB plugin's compiled module which only works with the system python
+ // (namely not Homebrew-installed python)
+ Path::new("/usr/bin/python3")
+ } else {
+ self.config
+ .python
+ .as_ref()
+ .expect("python is required for running LLDB or rustdoc tests")
+ }
+ }
+
+ /// Temporary directory that extended error information is emitted to.
+ fn extended_error_dir(&self) -> PathBuf {
+ self.out.join("tmp/extended-error-metadata")
+ }
+
+ /// Tests whether the `compiler` compiling for `target` should be forced to
+ /// use a stage1 compiler instead.
+ ///
+ /// Currently, by default, the build system does not perform a "full
+ /// bootstrap" by default where we compile the compiler three times.
+ /// Instead, we compile the compiler two times. The final stage (stage2)
+ /// just copies the libraries from the previous stage, which is what this
+ /// method detects.
+ ///
+ /// Here we return `true` if:
+ ///
+ /// * The build isn't performing a full bootstrap
+ /// * The `compiler` is in the final stage, 2
+ /// * We're not cross-compiling, so the artifacts are already available in
+ /// stage1
+ ///
+ /// When all of these conditions are met the build will lift artifacts from
+ /// the previous stage forward.
+ fn force_use_stage1(&self, compiler: Compiler, target: TargetSelection) -> bool {
+ !self.config.full_bootstrap
+ && compiler.stage >= 2
+ && (self.hosts.iter().any(|h| *h == target) || target == self.build)
+ }
+
+ /// Given `num` in the form "a.b.c" return a "release string" which
+ /// describes the release version number.
+ ///
+ /// For example on nightly this returns "a.b.c-nightly", on beta it returns
+ /// "a.b.c-beta.1" and on stable it just returns "a.b.c".
+ fn release(&self, num: &str) -> String {
+ match &self.config.channel[..] {
+ "stable" => num.to_string(),
+ "beta" => {
+ if self.rust_info.is_git() && !self.config.ignore_git {
+ format!("{}-beta.{}", num, self.beta_prerelease_version())
+ } else {
+ format!("{}-beta", num)
+ }
+ }
+ "nightly" => format!("{}-nightly", num),
+ _ => format!("{}-dev", num),
+ }
+ }
+
+ fn beta_prerelease_version(&self) -> u32 {
+ if let Some(s) = self.prerelease_version.get() {
+ return s;
+ }
+
+ // Figure out how many merge commits happened since we branched off master.
+ // That's our beta number!
+ // (Note that we use a `..` range, not the `...` symmetric difference.)
+ let count =
+ output(self.config.git().arg("rev-list").arg("--count").arg("--merges").arg(format!(
+ "refs/remotes/origin/{}..HEAD",
+ self.config.stage0_metadata.config.nightly_branch
+ )));
+ let n = count.trim().parse().unwrap();
+ self.prerelease_version.set(Some(n));
+ n
+ }
+
+ /// Returns the value of `release` above for Rust itself.
+ fn rust_release(&self) -> String {
+ self.release(&self.version)
+ }
+
+ /// Returns the "package version" for a component given the `num` release
+ /// number.
+ ///
+ /// The package version is typically what shows up in the names of tarballs.
+ /// For channels like beta/nightly it's just the channel name, otherwise
+ /// it's the `num` provided.
+ fn package_vers(&self, num: &str) -> String {
+ match &self.config.channel[..] {
+ "stable" => num.to_string(),
+ "beta" => "beta".to_string(),
+ "nightly" => "nightly".to_string(),
+ _ => format!("{}-dev", num),
+ }
+ }
+
+ /// Returns the value of `package_vers` above for Rust itself.
+ fn rust_package_vers(&self) -> String {
+ self.package_vers(&self.version)
+ }
+
+ fn llvm_link_tools_dynamically(&self, target: TargetSelection) -> bool {
+ target.contains("linux-gnu") || target.contains("apple-darwin")
+ }
+
+ /// Returns the `version` string associated with this compiler for Rust
+ /// itself.
+ ///
+ /// Note that this is a descriptive string which includes the commit date,
+ /// sha, version, etc.
+ fn rust_version(&self) -> String {
+ let mut version = self.rust_info.version(self, &self.version);
+ if let Some(ref s) = self.config.description {
+ version.push_str(" (");
+ version.push_str(s);
+ version.push(')');
+ }
+ version
+ }
+
+ /// Returns the full commit hash.
+ fn rust_sha(&self) -> Option<&str> {
+ self.rust_info.sha()
+ }
+
+ /// Returns the `a.b.c` version that the given package is at.
+ fn release_num(&self, package: &str) -> String {
+ let toml_file_name = self.src.join(&format!("src/tools/{}/Cargo.toml", package));
+ let toml = t!(fs::read_to_string(&toml_file_name));
+ for line in toml.lines() {
+ if let Some(stripped) =
+ line.strip_prefix("version = \"").and_then(|s| s.strip_suffix("\""))
+ {
+ return stripped.to_owned();
+ }
+ }
+
+ panic!("failed to find version in {}'s Cargo.toml", package)
+ }
+
+ /// Returns `true` if unstable features should be enabled for the compiler
+ /// we're building.
+ fn unstable_features(&self) -> bool {
+ match &self.config.channel[..] {
+ "stable" | "beta" => false,
+ "nightly" | _ => true,
+ }
+ }
+
+ /// Returns a Vec of all the dependencies of the given root crate,
+ /// including transitive dependencies and the root itself. Only includes
+ /// "local" crates (those in the local source tree, not from a registry).
+ fn in_tree_crates(&self, root: &str, target: Option<TargetSelection>) -> Vec<&Crate> {
+ let mut ret = Vec::new();
+ let mut list = vec![INTERNER.intern_str(root)];
+ let mut visited = HashSet::new();
+ while let Some(krate) = list.pop() {
+ let krate = &self.crates[&krate];
+ ret.push(krate);
+ for dep in &krate.deps {
+ if !self.crates.contains_key(dep) {
+ // Ignore non-workspace members.
+ continue;
+ }
+ // Don't include optional deps if their features are not
+ // enabled. Ideally this would be computed from `cargo
+ // metadata --features …`, but that is somewhat slow. In
+ // the future, we may want to consider just filtering all
+ // build and dev dependencies in metadata::build.
+ if visited.insert(dep)
+ && (dep != "profiler_builtins"
+ || target
+ .map(|t| self.config.profiler_enabled(t))
+ .unwrap_or_else(|| self.config.any_profiler_enabled()))
+ && (dep != "rustc_codegen_llvm" || self.config.llvm_enabled())
+ {
+ list.push(*dep);
+ }
+ }
+ }
+ ret
+ }
+
+ fn read_stamp_file(&self, stamp: &Path) -> Vec<(PathBuf, DependencyType)> {
+ if self.config.dry_run {
+ return Vec::new();
+ }
+
+ let mut paths = Vec::new();
+ let contents = t!(fs::read(stamp), &stamp);
+ // This is the method we use for extracting paths from the stamp file passed to us. See
+ // run_cargo for more information (in compile.rs).
+ for part in contents.split(|b| *b == 0) {
+ if part.is_empty() {
+ continue;
+ }
+ let dependency_type = match part[0] as char {
+ 'h' => DependencyType::Host,
+ 's' => DependencyType::TargetSelfContained,
+ 't' => DependencyType::Target,
+ _ => unreachable!(),
+ };
+ let path = PathBuf::from(t!(str::from_utf8(&part[1..])));
+ paths.push((path, dependency_type));
+ }
+ paths
+ }
+
+ /// Create a temporary directory in `out` and return its path.
+ ///
+ /// NOTE: this temporary directory is shared between all steps;
+ /// if you need an empty directory, create a new subdirectory inside it.
+ fn tempdir(&self) -> PathBuf {
+ let tmp = self.out.join("tmp");
+ t!(fs::create_dir_all(&tmp));
+ tmp
+ }
+
+ /// Copies a file from `src` to `dst`
+ pub fn copy(&self, src: &Path, dst: &Path) {
+ self.copy_internal(src, dst, false);
+ }
+
+ fn copy_internal(&self, src: &Path, dst: &Path, dereference_symlinks: bool) {
+ if self.config.dry_run {
+ return;
+ }
+ self.verbose_than(1, &format!("Copy {:?} to {:?}", src, dst));
+ if src == dst {
+ return;
+ }
+ let _ = fs::remove_file(&dst);
+ let metadata = t!(src.symlink_metadata());
+ let mut src = src.to_path_buf();
+ if metadata.file_type().is_symlink() {
+ if dereference_symlinks {
+ src = t!(fs::canonicalize(src));
+ } else {
+ let link = t!(fs::read_link(src));
+ t!(self.symlink_file(link, dst));
+ return;
+ }
+ }
+ if let Ok(()) = fs::hard_link(&src, dst) {
+ // Attempt to "easy copy" by creating a hard link
+ // (symlinks don't work on windows), but if that fails
+ // just fall back to a slow `copy` operation.
+ } else {
+ if let Err(e) = fs::copy(&src, dst) {
+ panic!("failed to copy `{}` to `{}`: {}", src.display(), dst.display(), e)
+ }
+ t!(fs::set_permissions(dst, metadata.permissions()));
+ let atime = FileTime::from_last_access_time(&metadata);
+ let mtime = FileTime::from_last_modification_time(&metadata);
+ t!(filetime::set_file_times(dst, atime, mtime));
+ }
+ }
+
+ /// Copies the `src` directory recursively to `dst`. Both are assumed to exist
+ /// when this function is called.
+ pub fn cp_r(&self, src: &Path, dst: &Path) {
+ if self.config.dry_run {
+ return;
+ }
+ for f in self.read_dir(src) {
+ let path = f.path();
+ let name = path.file_name().unwrap();
+ let dst = dst.join(name);
+ if t!(f.file_type()).is_dir() {
+ t!(fs::create_dir_all(&dst));
+ self.cp_r(&path, &dst);
+ } else {
+ let _ = fs::remove_file(&dst);
+ self.copy(&path, &dst);
+ }
+ }
+ }
+
+ /// Copies the `src` directory recursively to `dst`. Both are assumed to exist
+ /// when this function is called. Unwanted files or directories can be skipped
+ /// by returning `false` from the filter function.
+ pub fn cp_filtered(&self, src: &Path, dst: &Path, filter: &dyn Fn(&Path) -> bool) {
+ // Immediately recurse with an empty relative path
+ self.recurse_(src, dst, Path::new(""), filter)
+ }
+
+ // Inner function does the actual work
+ fn recurse_(&self, src: &Path, dst: &Path, relative: &Path, filter: &dyn Fn(&Path) -> bool) {
+ for f in self.read_dir(src) {
+ let path = f.path();
+ let name = path.file_name().unwrap();
+ let dst = dst.join(name);
+ let relative = relative.join(name);
+ // Only copy file or directory if the filter function returns true
+ if filter(&relative) {
+ if t!(f.file_type()).is_dir() {
+ let _ = fs::remove_dir_all(&dst);
+ self.create_dir(&dst);
+ self.recurse_(&path, &dst, &relative, filter);
+ } else {
+ let _ = fs::remove_file(&dst);
+ self.copy(&path, &dst);
+ }
+ }
+ }
+ }
+
+ fn copy_to_folder(&self, src: &Path, dest_folder: &Path) {
+ let file_name = src.file_name().unwrap();
+ let dest = dest_folder.join(file_name);
+ self.copy(src, &dest);
+ }
+
+ fn install(&self, src: &Path, dstdir: &Path, perms: u32) {
+ if self.config.dry_run {
+ return;
+ }
+ let dst = dstdir.join(src.file_name().unwrap());
+ self.verbose_than(1, &format!("Install {:?} to {:?}", src, dst));
+ t!(fs::create_dir_all(dstdir));
+ if !src.exists() {
+ panic!("Error: File \"{}\" not found!", src.display());
+ }
+ self.copy_internal(src, &dst, true);
+ chmod(&dst, perms);
+ }
+
+ fn create(&self, path: &Path, s: &str) {
+ if self.config.dry_run {
+ return;
+ }
+ t!(fs::write(path, s));
+ }
+
+ fn read(&self, path: &Path) -> String {
+ if self.config.dry_run {
+ return String::new();
+ }
+ t!(fs::read_to_string(path))
+ }
+
+ fn create_dir(&self, dir: &Path) {
+ if self.config.dry_run {
+ return;
+ }
+ t!(fs::create_dir_all(dir))
+ }
+
+ fn remove_dir(&self, dir: &Path) {
+ if self.config.dry_run {
+ return;
+ }
+ t!(fs::remove_dir_all(dir))
+ }
+
+ fn read_dir(&self, dir: &Path) -> impl Iterator<Item = fs::DirEntry> {
+ let iter = match fs::read_dir(dir) {
+ Ok(v) => v,
+ Err(_) if self.config.dry_run => return vec![].into_iter(),
+ Err(err) => panic!("could not read dir {:?}: {:?}", dir, err),
+ };
+ iter.map(|e| t!(e)).collect::<Vec<_>>().into_iter()
+ }
+
+ fn symlink_file<P: AsRef<Path>, Q: AsRef<Path>>(&self, src: P, link: Q) -> io::Result<()> {
+ #[cfg(unix)]
+ use std::os::unix::fs::symlink as symlink_file;
+ #[cfg(windows)]
+ use std::os::windows::fs::symlink_file;
+ if !self.config.dry_run { symlink_file(src.as_ref(), link.as_ref()) } else { Ok(()) }
+ }
+
+ fn remove(&self, f: &Path) {
+ if self.config.dry_run {
+ return;
+ }
+ fs::remove_file(f).unwrap_or_else(|_| panic!("failed to remove {:?}", f));
+ }
+
+ /// Returns if config.ninja is enabled, and checks for ninja existence,
+ /// exiting with a nicer error message if not.
+ fn ninja(&self) -> bool {
+ let mut cmd_finder = crate::sanity::Finder::new();
+
+ if self.config.ninja_in_file {
+ // Some Linux distros rename `ninja` to `ninja-build`.
+ // CMake can work with either binary name.
+ if cmd_finder.maybe_have("ninja-build").is_none()
+ && cmd_finder.maybe_have("ninja").is_none()
+ {
+ eprintln!(
+ "
+Couldn't find required command: ninja (or ninja-build)
+
+You should install ninja as described at
+<https://github.com/ninja-build/ninja/wiki/Pre-built-Ninja-packages>,
+or set `ninja = false` in the `[llvm]` section of `config.toml`.
+Alternatively, set `download-ci-llvm = true` in that `[llvm]` section
+to download LLVM rather than building it.
+"
+ );
+ detail_exit(1);
+ }
+ }
+
+ // If ninja isn't enabled but we're building for MSVC then we try
+ // doubly hard to enable it. It was realized in #43767 that the msbuild
+ // CMake generator for MSVC doesn't respect configuration options like
+ // disabling LLVM assertions, which can often be quite important!
+ //
+ // In these cases we automatically enable Ninja if we find it in the
+ // environment.
+ if !self.config.ninja_in_file && self.config.build.contains("msvc") {
+ if cmd_finder.maybe_have("ninja").is_some() {
+ return true;
+ }
+ }
+
+ self.config.ninja_in_file
+ }
+}
+
+#[cfg(unix)]
+fn chmod(path: &Path, perms: u32) {
+ use std::os::unix::fs::*;
+ t!(fs::set_permissions(path, fs::Permissions::from_mode(perms)));
+}
+#[cfg(windows)]
+fn chmod(_path: &Path, _perms: u32) {}
+
+/// If code is not 0 (successful exit status), exit status is 101 (rust's default error code.)
+/// If the test is running and code is an error code, it will cause a panic.
+fn detail_exit(code: i32) -> ! {
+ // Successful exit
+ if code == 0 {
+ std::process::exit(0);
+ }
+ if cfg!(test) {
+ panic!("status code: {}", code);
+ } else {
+ std::panic::resume_unwind(Box::new(code));
+ }
+}
+
+impl Compiler {
+ pub fn with_stage(mut self, stage: u32) -> Compiler {
+ self.stage = stage;
+ self
+ }
+
+ /// Returns `true` if this is a snapshot compiler for `build`'s configuration
+ pub fn is_snapshot(&self, build: &Build) -> bool {
+ self.stage == 0 && self.host == build.build
+ }
+
+ /// Returns if this compiler should be treated as a final stage one in the
+ /// current build session.
+ /// This takes into account whether we're performing a full bootstrap or
+ /// not; don't directly compare the stage with `2`!
+ pub fn is_final_stage(&self, build: &Build) -> bool {
+ let final_stage = if build.config.full_bootstrap { 2 } else { 1 };
+ self.stage >= final_stage
+ }
+}
+
+fn envify(s: &str) -> String {
+ s.chars()
+ .map(|c| match c {
+ '-' => '_',
+ c => c,
+ })
+ .flat_map(|c| c.to_uppercase())
+ .collect()
+}
diff --git a/src/bootstrap/metadata.rs b/src/bootstrap/metadata.rs
new file mode 100644
index 000000000..e193e70a0
--- /dev/null
+++ b/src/bootstrap/metadata.rs
@@ -0,0 +1,59 @@
+use std::path::PathBuf;
+use std::process::Command;
+
+use serde::Deserialize;
+
+use crate::cache::INTERNER;
+use crate::util::output;
+use crate::{Build, Crate};
+
+#[derive(Deserialize)]
+struct Output {
+ packages: Vec<Package>,
+}
+
+#[derive(Deserialize)]
+struct Package {
+ name: String,
+ source: Option<String>,
+ manifest_path: String,
+ dependencies: Vec<Dependency>,
+}
+
+#[derive(Deserialize)]
+struct Dependency {
+ name: String,
+ source: Option<String>,
+}
+
+pub fn build(build: &mut Build) {
+ // Run `cargo metadata` to figure out what crates we're testing.
+ let mut cargo = Command::new(&build.initial_cargo);
+ cargo
+ .arg("metadata")
+ .arg("--format-version")
+ .arg("1")
+ .arg("--no-deps")
+ .arg("--manifest-path")
+ .arg(build.src.join("Cargo.toml"));
+ let output = output(&mut cargo);
+ let output: Output = serde_json::from_str(&output).unwrap();
+ for package in output.packages {
+ if package.source.is_none() {
+ let name = INTERNER.intern_string(package.name);
+ let mut path = PathBuf::from(package.manifest_path);
+ path.pop();
+ let deps = package
+ .dependencies
+ .into_iter()
+ .filter(|dep| dep.source.is_none())
+ .map(|dep| INTERNER.intern_string(dep.name))
+ .collect();
+ let krate = Crate { name, deps, path };
+ let relative_path = krate.local_path(build);
+ build.crates.insert(name, krate);
+ let existing_path = build.crate_paths.insert(relative_path, name);
+ assert!(existing_path.is_none(), "multiple crates with the same path");
+ }
+ }
+}
diff --git a/src/bootstrap/metrics.rs b/src/bootstrap/metrics.rs
new file mode 100644
index 000000000..451febddc
--- /dev/null
+++ b/src/bootstrap/metrics.rs
@@ -0,0 +1,208 @@
+//! This module is responsible for collecting metrics profiling information for the current build
+//! and dumping it to disk as JSON, to aid investigations on build and CI performance.
+//!
+//! As this module requires additional dependencies not present during local builds, it's cfg'd
+//! away whenever the `build.metrics` config option is not set to `true`.
+
+use crate::builder::Step;
+use crate::util::t;
+use crate::Build;
+use serde::{Deserialize, Serialize};
+use std::cell::RefCell;
+use std::fs::File;
+use std::io::BufWriter;
+use std::time::{Duration, Instant};
+use sysinfo::{CpuExt, System, SystemExt};
+
+pub(crate) struct BuildMetrics {
+ state: RefCell<MetricsState>,
+}
+
+impl BuildMetrics {
+ pub(crate) fn init() -> Self {
+ let state = RefCell::new(MetricsState {
+ finished_steps: Vec::new(),
+ running_steps: Vec::new(),
+
+ system_info: System::new(),
+ timer_start: None,
+ invocation_timer_start: Instant::now(),
+ });
+
+ BuildMetrics { state }
+ }
+
+ pub(crate) fn enter_step<S: Step>(&self, step: &S) {
+ let mut state = self.state.borrow_mut();
+
+ // Consider all the stats gathered so far as the parent's.
+ if !state.running_steps.is_empty() {
+ self.collect_stats(&mut *state);
+ }
+
+ state.system_info.refresh_cpu();
+ state.timer_start = Some(Instant::now());
+
+ state.running_steps.push(StepMetrics {
+ type_: std::any::type_name::<S>().into(),
+ debug_repr: format!("{step:?}"),
+
+ cpu_usage_time_sec: 0.0,
+ duration_excluding_children_sec: Duration::ZERO,
+
+ children: Vec::new(),
+ });
+ }
+
+ pub(crate) fn exit_step(&self) {
+ let mut state = self.state.borrow_mut();
+
+ self.collect_stats(&mut *state);
+
+ let step = state.running_steps.pop().unwrap();
+ if state.running_steps.is_empty() {
+ state.finished_steps.push(step);
+ state.timer_start = None;
+ } else {
+ state.running_steps.last_mut().unwrap().children.push(step);
+
+ // Start collecting again for the parent step.
+ state.system_info.refresh_cpu();
+ state.timer_start = Some(Instant::now());
+ }
+ }
+
+ fn collect_stats(&self, state: &mut MetricsState) {
+ let step = state.running_steps.last_mut().unwrap();
+
+ let elapsed = state.timer_start.unwrap().elapsed();
+ step.duration_excluding_children_sec += elapsed;
+
+ state.system_info.refresh_cpu();
+ let cpu = state.system_info.cpus().iter().map(|p| p.cpu_usage()).sum::<f32>();
+ step.cpu_usage_time_sec += cpu as f64 / 100.0 * elapsed.as_secs_f64();
+ }
+
+ pub(crate) fn persist(&self, build: &Build) {
+ let mut state = self.state.borrow_mut();
+ assert!(state.running_steps.is_empty(), "steps are still executing");
+
+ let dest = build.out.join("metrics.json");
+
+ let mut system = System::new();
+ system.refresh_cpu();
+ system.refresh_memory();
+
+ let system_stats = JsonInvocationSystemStats {
+ cpu_threads_count: system.cpus().len(),
+ cpu_model: system.cpus()[0].brand().into(),
+
+ memory_total_bytes: system.total_memory() * 1024,
+ };
+ let steps = std::mem::take(&mut state.finished_steps);
+
+ // Some of our CI builds consist of multiple independent CI invocations. Ensure all the
+ // previous invocations are still present in the resulting file.
+ let mut invocations = match std::fs::read(&dest) {
+ Ok(contents) => t!(serde_json::from_slice::<JsonRoot>(&contents)).invocations,
+ Err(err) => {
+ if err.kind() != std::io::ErrorKind::NotFound {
+ panic!("failed to open existing metrics file at {}: {err}", dest.display());
+ }
+ Vec::new()
+ }
+ };
+ invocations.push(JsonInvocation {
+ duration_including_children_sec: state.invocation_timer_start.elapsed().as_secs_f64(),
+ children: steps.into_iter().map(|step| self.prepare_json_step(step)).collect(),
+ });
+
+ let json = JsonRoot { system_stats, invocations };
+
+ t!(std::fs::create_dir_all(dest.parent().unwrap()));
+ let mut file = BufWriter::new(t!(File::create(&dest)));
+ t!(serde_json::to_writer(&mut file, &json));
+ }
+
+ fn prepare_json_step(&self, step: StepMetrics) -> JsonNode {
+ JsonNode::RustbuildStep {
+ type_: step.type_,
+ debug_repr: step.debug_repr,
+
+ duration_excluding_children_sec: step.duration_excluding_children_sec.as_secs_f64(),
+ system_stats: JsonStepSystemStats {
+ cpu_utilization_percent: step.cpu_usage_time_sec * 100.0
+ / step.duration_excluding_children_sec.as_secs_f64(),
+ },
+
+ children: step
+ .children
+ .into_iter()
+ .map(|child| self.prepare_json_step(child))
+ .collect(),
+ }
+ }
+}
+
+struct MetricsState {
+ finished_steps: Vec<StepMetrics>,
+ running_steps: Vec<StepMetrics>,
+
+ system_info: System,
+ timer_start: Option<Instant>,
+ invocation_timer_start: Instant,
+}
+
+struct StepMetrics {
+ type_: String,
+ debug_repr: String,
+
+ cpu_usage_time_sec: f64,
+ duration_excluding_children_sec: Duration,
+
+ children: Vec<StepMetrics>,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+struct JsonRoot {
+ system_stats: JsonInvocationSystemStats,
+ invocations: Vec<JsonInvocation>,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+struct JsonInvocation {
+ duration_including_children_sec: f64,
+ children: Vec<JsonNode>,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(tag = "kind", rename_all = "snake_case")]
+enum JsonNode {
+ RustbuildStep {
+ #[serde(rename = "type")]
+ type_: String,
+ debug_repr: String,
+
+ duration_excluding_children_sec: f64,
+ system_stats: JsonStepSystemStats,
+
+ children: Vec<JsonNode>,
+ },
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+struct JsonInvocationSystemStats {
+ cpu_threads_count: usize,
+ cpu_model: String,
+
+ memory_total_bytes: u64,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+struct JsonStepSystemStats {
+ cpu_utilization_percent: f64,
+}
diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in
new file mode 100644
index 000000000..5a1f2e704
--- /dev/null
+++ b/src/bootstrap/mk/Makefile.in
@@ -0,0 +1,83 @@
+ifdef VERBOSE
+Q :=
+BOOTSTRAP_ARGS := -v
+else
+Q := @
+BOOTSTRAP_ARGS :=
+endif
+
+BOOTSTRAP := $(CFG_PYTHON) $(CFG_SRC_DIR)src/bootstrap/bootstrap.py
+
+all:
+ $(Q)$(BOOTSTRAP) build --stage 2 $(BOOTSTRAP_ARGS)
+ $(Q)$(BOOTSTRAP) doc --stage 2 $(BOOTSTRAP_ARGS)
+
+help:
+ $(Q)echo 'Welcome to the rustbuild build system!'
+ $(Q)echo
+ $(Q)echo This makefile is a thin veneer over the ./x.py script located
+ $(Q)echo in this directory. To get the full power of the build system
+ $(Q)echo you can run x.py directly.
+ $(Q)echo
+ $(Q)echo To learn more run \`./x.py --help\`
+
+clean:
+ $(Q)$(BOOTSTRAP) clean $(BOOTSTRAP_ARGS)
+
+rustc-stage1:
+ $(Q)$(BOOTSTRAP) build --stage 1 library/test $(BOOTSTRAP_ARGS)
+rustc-stage2:
+ $(Q)$(BOOTSTRAP) build --stage 2 library/test $(BOOTSTRAP_ARGS)
+
+docs: doc
+doc:
+ $(Q)$(BOOTSTRAP) doc --stage 2 $(BOOTSTRAP_ARGS)
+nomicon:
+ $(Q)$(BOOTSTRAP) doc --stage 2 src/doc/nomicon $(BOOTSTRAP_ARGS)
+book:
+ $(Q)$(BOOTSTRAP) doc --stage 2 src/doc/book $(BOOTSTRAP_ARGS)
+standalone-docs:
+ $(Q)$(BOOTSTRAP) doc --stage 2 src/doc $(BOOTSTRAP_ARGS)
+check:
+ $(Q)$(BOOTSTRAP) test --stage 2 $(BOOTSTRAP_ARGS)
+check-aux:
+ $(Q)$(BOOTSTRAP) test --stage 2 \
+ src/tools/cargo \
+ src/tools/cargotest \
+ $(BOOTSTRAP_ARGS)
+dist:
+ $(Q)$(BOOTSTRAP) dist $(BOOTSTRAP_ARGS)
+distcheck:
+ $(Q)$(BOOTSTRAP) dist $(BOOTSTRAP_ARGS)
+ $(Q)$(BOOTSTRAP) test --stage 2 distcheck $(BOOTSTRAP_ARGS)
+install:
+ $(Q)$(BOOTSTRAP) install $(BOOTSTRAP_ARGS)
+tidy:
+ $(Q)$(BOOTSTRAP) test --stage 2 src/tools/tidy $(BOOTSTRAP_ARGS)
+prepare:
+ $(Q)$(BOOTSTRAP) build --stage 2 nonexistent/path/to/trigger/cargo/metadata
+
+check-stage2-T-arm-linux-androideabi-H-x86_64-unknown-linux-gnu:
+ $(Q)$(BOOTSTRAP) test --stage 2 --target arm-linux-androideabi
+check-stage2-T-x86_64-unknown-linux-musl-H-x86_64-unknown-linux-gnu:
+ $(Q)$(BOOTSTRAP) test --stage 2 --target x86_64-unknown-linux-musl
+
+TESTS_IN_2 := \
+ src/test/ui \
+ src/tools/linkchecker
+
+ci-subset-1:
+ $(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_2:%=--exclude %)
+ci-subset-2:
+ $(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_2)
+
+TESTS_IN_MINGW_2 := \
+ src/test/ui
+
+ci-mingw-subset-1:
+ $(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_MINGW_2:%=--exclude %)
+ci-mingw-subset-2:
+ $(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_MINGW_2)
+
+
+.PHONY: dist
diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs
new file mode 100644
index 000000000..4d548dbb6
--- /dev/null
+++ b/src/bootstrap/native.rs
@@ -0,0 +1,1335 @@
+//! Compilation of native dependencies like LLVM.
+//!
+//! Native projects like LLVM unfortunately aren't suited just yet for
+//! compilation in build scripts that Cargo has. This is because the
+//! compilation takes a *very* long time but also because we don't want to
+//! compile LLVM 3 times as part of a normal bootstrap (we want it cached).
+//!
+//! LLVM and compiler-rt are essentially just wired up to everything else to
+//! ensure that they're always in place if needed.
+
+use std::env;
+use std::env::consts::EXE_EXTENSION;
+use std::ffi::{OsStr, OsString};
+use std::fs::{self, File};
+use std::io;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
+use crate::builder::{Builder, RunConfig, ShouldRun, Step};
+use crate::config::TargetSelection;
+use crate::util::get_clang_cl_resource_dir;
+use crate::util::{self, exe, output, program_out_of_date, t, up_to_date};
+use crate::{CLang, GitRepo};
+
+pub struct Meta {
+ stamp: HashStamp,
+ build_llvm_config: PathBuf,
+ out_dir: PathBuf,
+ root: String,
+}
+
+// Linker flags to pass to LLVM's CMake invocation.
+#[derive(Debug, Clone, Default)]
+struct LdFlags {
+ // CMAKE_EXE_LINKER_FLAGS
+ exe: OsString,
+ // CMAKE_SHARED_LINKER_FLAGS
+ shared: OsString,
+ // CMAKE_MODULE_LINKER_FLAGS
+ module: OsString,
+}
+
+impl LdFlags {
+ fn push_all(&mut self, s: impl AsRef<OsStr>) {
+ let s = s.as_ref();
+ self.exe.push(" ");
+ self.exe.push(s);
+ self.shared.push(" ");
+ self.shared.push(s);
+ self.module.push(" ");
+ self.module.push(s);
+ }
+}
+
+// This returns whether we've already previously built LLVM.
+//
+// It's used to avoid busting caches during x.py check -- if we've already built
+// LLVM, it's fine for us to not try to avoid doing so.
+//
+// This will return the llvm-config if it can get it (but it will not build it
+// if not).
+pub fn prebuilt_llvm_config(
+ builder: &Builder<'_>,
+ target: TargetSelection,
+) -> Result<PathBuf, Meta> {
+ maybe_download_ci_llvm(builder);
+
+ // If we're using a custom LLVM bail out here, but we can only use a
+ // custom LLVM for the build triple.
+ if let Some(config) = builder.config.target_config.get(&target) {
+ if let Some(ref s) = config.llvm_config {
+ check_llvm_version(builder, s);
+ return Ok(s.to_path_buf());
+ }
+ }
+
+ let root = "src/llvm-project/llvm";
+ let out_dir = builder.llvm_out(target);
+
+ let mut llvm_config_ret_dir = builder.llvm_out(builder.config.build);
+ if !builder.config.build.contains("msvc") || builder.ninja() {
+ llvm_config_ret_dir.push("build");
+ }
+ llvm_config_ret_dir.push("bin");
+
+ let build_llvm_config = llvm_config_ret_dir.join(exe("llvm-config", builder.config.build));
+
+ let stamp = out_dir.join("llvm-finished-building");
+ let stamp = HashStamp::new(stamp, builder.in_tree_llvm_info.sha());
+
+ if builder.config.llvm_skip_rebuild && stamp.path.exists() {
+ builder.info(
+ "Warning: \
+ Using a potentially stale build of LLVM; \
+ This may not behave well.",
+ );
+ return Ok(build_llvm_config);
+ }
+
+ if stamp.is_done() {
+ if stamp.hash.is_none() {
+ builder.info(
+ "Could not determine the LLVM submodule commit hash. \
+ Assuming that an LLVM rebuild is not necessary.",
+ );
+ builder.info(&format!(
+ "To force LLVM to rebuild, remove the file `{}`",
+ stamp.path.display()
+ ));
+ }
+ return Ok(build_llvm_config);
+ }
+
+ Err(Meta { stamp, build_llvm_config, out_dir, root: root.into() })
+}
+
+pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) {
+ let config = &builder.config;
+ if !config.llvm_from_ci {
+ return;
+ }
+ let mut rev_list = config.git();
+ rev_list.args(&[
+ PathBuf::from("rev-list"),
+ format!("--author={}", builder.config.stage0_metadata.config.git_merge_commit_email).into(),
+ "-n1".into(),
+ "--first-parent".into(),
+ "HEAD".into(),
+ "--".into(),
+ builder.src.join("src/llvm-project"),
+ builder.src.join("src/bootstrap/download-ci-llvm-stamp"),
+ // the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
+ builder.src.join("src/version"),
+ ]);
+ let llvm_sha = output(&mut rev_list);
+ let llvm_sha = llvm_sha.trim();
+
+ if llvm_sha == "" {
+ eprintln!("error: could not find commit hash for downloading LLVM");
+ eprintln!("help: maybe your repository history is too shallow?");
+ eprintln!("help: consider disabling `download-ci-llvm`");
+ eprintln!("help: or fetch enough history to include one upstream commit");
+ panic!();
+ }
+
+ let llvm_root = config.ci_llvm_root();
+ let llvm_stamp = llvm_root.join(".llvm-stamp");
+ let key = format!("{}{}", llvm_sha, config.llvm_assertions);
+ if program_out_of_date(&llvm_stamp, &key) && !config.dry_run {
+ download_ci_llvm(builder, &llvm_sha);
+ for entry in t!(fs::read_dir(llvm_root.join("bin"))) {
+ builder.fix_bin_or_dylib(&t!(entry).path());
+ }
+
+ // Update the timestamp of llvm-config to force rustc_llvm to be
+ // rebuilt. This is a hacky workaround for a deficiency in Cargo where
+ // the rerun-if-changed directive doesn't handle changes very well.
+ // https://github.com/rust-lang/cargo/issues/10791
+ // Cargo only compares the timestamp of the file relative to the last
+ // time `rustc_llvm` build script ran. However, the timestamps of the
+ // files in the tarball are in the past, so it doesn't trigger a
+ // rebuild.
+ let now = filetime::FileTime::from_system_time(std::time::SystemTime::now());
+ let llvm_config = llvm_root.join("bin").join(exe("llvm-config", builder.config.build));
+ t!(filetime::set_file_times(&llvm_config, now, now));
+
+ let llvm_lib = llvm_root.join("lib");
+ for entry in t!(fs::read_dir(&llvm_lib)) {
+ let lib = t!(entry).path();
+ if lib.extension().map_or(false, |ext| ext == "so") {
+ builder.fix_bin_or_dylib(&lib);
+ }
+ }
+ t!(fs::write(llvm_stamp, key));
+ }
+}
+
+fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) {
+ let llvm_assertions = builder.config.llvm_assertions;
+
+ let cache_prefix = format!("llvm-{}-{}", llvm_sha, llvm_assertions);
+ let cache_dst = builder.out.join("cache");
+ let rustc_cache = cache_dst.join(cache_prefix);
+ if !rustc_cache.exists() {
+ t!(fs::create_dir_all(&rustc_cache));
+ }
+ let base = if llvm_assertions {
+ &builder.config.stage0_metadata.config.artifacts_with_llvm_assertions_server
+ } else {
+ &builder.config.stage0_metadata.config.artifacts_server
+ };
+ let channel = builder.config.artifact_channel(llvm_sha);
+ let filename = format!("rust-dev-{}-{}.tar.xz", channel, builder.build.build.triple);
+ let tarball = rustc_cache.join(&filename);
+ if !tarball.exists() {
+ let help_on_error = "error: failed to download llvm from ci
+
+help: old builds get deleted after a certain time
+help: if trying to compile an old commit of rustc, disable `download-ci-llvm` in config.toml:
+
+[llvm]
+download-ci-llvm = false
+";
+ builder.download_component(
+ &format!("{base}/{llvm_sha}/{filename}"),
+ &tarball,
+ help_on_error,
+ );
+ }
+ let llvm_root = builder.config.ci_llvm_root();
+ builder.unpack(&tarball, &llvm_root, "rust-dev");
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Llvm {
+ pub target: TargetSelection,
+}
+
+impl Step for Llvm {
+ type Output = PathBuf; // path to llvm-config
+
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/llvm-project").path("src/llvm-project/llvm")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Llvm { target: run.target });
+ }
+
+ /// Compile LLVM for `target`.
+ fn run(self, builder: &Builder<'_>) -> PathBuf {
+ let target = self.target;
+ let target_native = if self.target.starts_with("riscv") {
+ // RISC-V target triples in Rust is not named the same as C compiler target triples.
+ // This converts Rust RISC-V target triples to C compiler triples.
+ let idx = target.triple.find('-').unwrap();
+
+ format!("riscv{}{}", &target.triple[5..7], &target.triple[idx..])
+ } else if self.target.starts_with("powerpc") && self.target.ends_with("freebsd") {
+ // FreeBSD 13 had incompatible ABI changes on all PowerPC platforms.
+ // Set the version suffix to 13.0 so the correct target details are used.
+ format!("{}{}", self.target, "13.0")
+ } else {
+ target.to_string()
+ };
+
+ let Meta { stamp, build_llvm_config, out_dir, root } =
+ match prebuilt_llvm_config(builder, target) {
+ Ok(p) => return p,
+ Err(m) => m,
+ };
+
+ builder.update_submodule(&Path::new("src").join("llvm-project"));
+ if builder.llvm_link_shared() && target.contains("windows") {
+ panic!("shared linking to LLVM is not currently supported on {}", target.triple);
+ }
+
+ builder.info(&format!("Building LLVM for {}", target));
+ t!(stamp.remove());
+ let _time = util::timeit(&builder);
+ t!(fs::create_dir_all(&out_dir));
+
+ // https://llvm.org/docs/CMake.html
+ let mut cfg = cmake::Config::new(builder.src.join(root));
+ let mut ldflags = LdFlags::default();
+
+ let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) {
+ (false, _) => "Debug",
+ (true, false) => "Release",
+ (true, true) => "RelWithDebInfo",
+ };
+
+ // NOTE: remember to also update `config.toml.example` when changing the
+ // defaults!
+ let llvm_targets = match &builder.config.llvm_targets {
+ Some(s) => s,
+ None => {
+ "AArch64;ARM;BPF;Hexagon;MSP430;Mips;NVPTX;PowerPC;RISCV;\
+ Sparc;SystemZ;WebAssembly;X86"
+ }
+ };
+
+ let llvm_exp_targets = match builder.config.llvm_experimental_targets {
+ Some(ref s) => s,
+ None => "AVR;M68k",
+ };
+
+ let assertions = if builder.config.llvm_assertions { "ON" } else { "OFF" };
+ let plugins = if builder.config.llvm_plugins { "ON" } else { "OFF" };
+ let enable_tests = if builder.config.llvm_tests { "ON" } else { "OFF" };
+
+ cfg.out_dir(&out_dir)
+ .profile(profile)
+ .define("LLVM_ENABLE_ASSERTIONS", assertions)
+ .define("LLVM_ENABLE_PLUGINS", plugins)
+ .define("LLVM_TARGETS_TO_BUILD", llvm_targets)
+ .define("LLVM_EXPERIMENTAL_TARGETS_TO_BUILD", llvm_exp_targets)
+ .define("LLVM_INCLUDE_EXAMPLES", "OFF")
+ .define("LLVM_INCLUDE_DOCS", "OFF")
+ .define("LLVM_INCLUDE_BENCHMARKS", "OFF")
+ .define("LLVM_INCLUDE_TESTS", enable_tests)
+ .define("LLVM_ENABLE_TERMINFO", "OFF")
+ .define("LLVM_ENABLE_LIBEDIT", "OFF")
+ .define("LLVM_ENABLE_BINDINGS", "OFF")
+ .define("LLVM_ENABLE_Z3_SOLVER", "OFF")
+ .define("LLVM_PARALLEL_COMPILE_JOBS", builder.jobs().to_string())
+ .define("LLVM_TARGET_ARCH", target_native.split('-').next().unwrap())
+ .define("LLVM_DEFAULT_TARGET_TRIPLE", target_native);
+
+ // Parts of our test suite rely on the `FileCheck` tool, which is built by default in
+ // `build/$TARGET/llvm/build/bin` is but *not* then installed to `build/$TARGET/llvm/bin`.
+ // This flag makes sure `FileCheck` is copied in the final binaries directory.
+ cfg.define("LLVM_INSTALL_UTILS", "ON");
+
+ if builder.config.llvm_profile_generate {
+ cfg.define("LLVM_BUILD_INSTRUMENTED", "IR");
+ if let Ok(llvm_profile_dir) = std::env::var("LLVM_PROFILE_DIR") {
+ cfg.define("LLVM_PROFILE_DATA_DIR", llvm_profile_dir);
+ }
+ cfg.define("LLVM_BUILD_RUNTIME", "No");
+ }
+ if let Some(path) = builder.config.llvm_profile_use.as_ref() {
+ cfg.define("LLVM_PROFDATA_FILE", &path);
+ }
+
+ if target != "aarch64-apple-darwin" && !target.contains("windows") {
+ cfg.define("LLVM_ENABLE_ZLIB", "ON");
+ } else {
+ cfg.define("LLVM_ENABLE_ZLIB", "OFF");
+ }
+
+ // Are we compiling for iOS/tvOS/watchOS?
+ if target.contains("apple-ios")
+ || target.contains("apple-tvos")
+ || target.contains("apple-watchos")
+ {
+ // These two defines prevent CMake from automatically trying to add a MacOSX sysroot, which leads to a compiler error.
+ cfg.define("CMAKE_OSX_SYSROOT", "/");
+ cfg.define("CMAKE_OSX_DEPLOYMENT_TARGET", "");
+ // Prevent cmake from adding -bundle to CFLAGS automatically, which leads to a compiler error because "-bitcode_bundle" also gets added.
+ cfg.define("LLVM_ENABLE_PLUGINS", "OFF");
+ // Zlib fails to link properly, leading to a compiler error.
+ cfg.define("LLVM_ENABLE_ZLIB", "OFF");
+ }
+
+ // This setting makes the LLVM tools link to the dynamic LLVM library,
+ // which saves both memory during parallel links and overall disk space
+ // for the tools. We don't do this on every platform as it doesn't work
+ // equally well everywhere.
+ //
+ // If we're not linking rustc to a dynamic LLVM, though, then don't link
+ // tools to it.
+ let llvm_link_shared =
+ builder.llvm_link_tools_dynamically(target) && builder.llvm_link_shared();
+ if llvm_link_shared {
+ cfg.define("LLVM_LINK_LLVM_DYLIB", "ON");
+ }
+
+ if target.starts_with("riscv") && !target.contains("freebsd") {
+ // RISC-V GCC erroneously requires linking against
+ // `libatomic` when using 1-byte and 2-byte C++
+ // atomics but the LLVM build system check cannot
+ // detect this. Therefore it is set manually here.
+ // FreeBSD uses Clang as its system compiler and
+ // provides no libatomic in its base system so does
+ // not want this.
+ ldflags.exe.push(" -latomic");
+ ldflags.shared.push(" -latomic");
+ }
+
+ if target.contains("msvc") {
+ cfg.define("LLVM_USE_CRT_DEBUG", "MT");
+ cfg.define("LLVM_USE_CRT_RELEASE", "MT");
+ cfg.define("LLVM_USE_CRT_RELWITHDEBINFO", "MT");
+ cfg.static_crt(true);
+ }
+
+ if target.starts_with("i686") {
+ cfg.define("LLVM_BUILD_32_BITS", "ON");
+ }
+
+ let mut enabled_llvm_projects = Vec::new();
+
+ if util::forcing_clang_based_tests() {
+ enabled_llvm_projects.push("clang");
+ enabled_llvm_projects.push("compiler-rt");
+ }
+
+ if builder.config.llvm_polly {
+ enabled_llvm_projects.push("polly");
+ }
+
+ if builder.config.llvm_clang {
+ enabled_llvm_projects.push("clang");
+ }
+
+ // We want libxml to be disabled.
+ // See https://github.com/rust-lang/rust/pull/50104
+ cfg.define("LLVM_ENABLE_LIBXML2", "OFF");
+
+ if !enabled_llvm_projects.is_empty() {
+ enabled_llvm_projects.sort();
+ enabled_llvm_projects.dedup();
+ cfg.define("LLVM_ENABLE_PROJECTS", enabled_llvm_projects.join(";"));
+ }
+
+ if let Some(num_linkers) = builder.config.llvm_link_jobs {
+ if num_linkers > 0 {
+ cfg.define("LLVM_PARALLEL_LINK_JOBS", num_linkers.to_string());
+ }
+ }
+
+ // Workaround for ppc32 lld limitation
+ if target == "powerpc-unknown-freebsd" {
+ ldflags.exe.push(" -fuse-ld=bfd");
+ }
+
+ // https://llvm.org/docs/HowToCrossCompileLLVM.html
+ if target != builder.config.build {
+ builder.ensure(Llvm { target: builder.config.build });
+ // FIXME: if the llvm root for the build triple is overridden then we
+ // should use llvm-tblgen from there, also should verify that it
+ // actually exists most of the time in normal installs of LLVM.
+ let host_bin = builder.llvm_out(builder.config.build).join("bin");
+ cfg.define("LLVM_TABLEGEN", host_bin.join("llvm-tblgen").with_extension(EXE_EXTENSION));
+ // LLVM_NM is required for cross compiling using MSVC
+ cfg.define("LLVM_NM", host_bin.join("llvm-nm").with_extension(EXE_EXTENSION));
+ cfg.define(
+ "LLVM_CONFIG_PATH",
+ host_bin.join("llvm-config").with_extension(EXE_EXTENSION),
+ );
+ if builder.config.llvm_clang {
+ let build_bin = builder.llvm_out(builder.config.build).join("build").join("bin");
+ let clang_tblgen = build_bin.join("clang-tblgen").with_extension(EXE_EXTENSION);
+ if !builder.config.dry_run && !clang_tblgen.exists() {
+ panic!("unable to find {}", clang_tblgen.display());
+ }
+ cfg.define("CLANG_TABLEGEN", clang_tblgen);
+ }
+ }
+
+ let llvm_version_suffix = if let Some(ref suffix) = builder.config.llvm_version_suffix {
+ // Allow version-suffix="" to not define a version suffix at all.
+ if !suffix.is_empty() { Some(suffix.to_string()) } else { None }
+ } else if builder.config.channel == "dev" {
+ // Changes to a version suffix require a complete rebuild of the LLVM.
+ // To avoid rebuilds during a time of version bump, don't include rustc
+ // release number on the dev channel.
+ Some("-rust-dev".to_string())
+ } else {
+ Some(format!("-rust-{}-{}", builder.version, builder.config.channel))
+ };
+ if let Some(ref suffix) = llvm_version_suffix {
+ cfg.define("LLVM_VERSION_SUFFIX", suffix);
+ }
+
+ configure_cmake(builder, target, &mut cfg, true, ldflags);
+ configure_llvm(builder, target, &mut cfg);
+
+ for (key, val) in &builder.config.llvm_build_config {
+ cfg.define(key, val);
+ }
+
+ // FIXME: we don't actually need to build all LLVM tools and all LLVM
+ // libraries here, e.g., we just want a few components and a few
+ // tools. Figure out how to filter them down and only build the right
+ // tools and libs on all platforms.
+
+ if builder.config.dry_run {
+ return build_llvm_config;
+ }
+
+ cfg.build();
+
+ // When building LLVM with LLVM_LINK_LLVM_DYLIB for macOS, an unversioned
+ // libLLVM.dylib will be built. However, llvm-config will still look
+ // for a versioned path like libLLVM-14.dylib. Manually create a symbolic
+ // link to make llvm-config happy.
+ if llvm_link_shared && target.contains("apple-darwin") {
+ let mut cmd = Command::new(&build_llvm_config);
+ let version = output(cmd.arg("--version"));
+ let major = version.split('.').next().unwrap();
+ let lib_name = match llvm_version_suffix {
+ Some(s) => format!("libLLVM-{}{}.dylib", major, s),
+ None => format!("libLLVM-{}.dylib", major),
+ };
+
+ let lib_llvm = out_dir.join("build").join("lib").join(lib_name);
+ if !lib_llvm.exists() {
+ t!(builder.symlink_file("libLLVM.dylib", &lib_llvm));
+ }
+ }
+
+ t!(stamp.write());
+
+ build_llvm_config
+ }
+}
+
+fn check_llvm_version(builder: &Builder<'_>, llvm_config: &Path) {
+ if !builder.config.llvm_version_check {
+ return;
+ }
+
+ if builder.config.dry_run {
+ return;
+ }
+
+ let mut cmd = Command::new(llvm_config);
+ let version = output(cmd.arg("--version"));
+ let mut parts = version.split('.').take(2).filter_map(|s| s.parse::<u32>().ok());
+ if let (Some(major), Some(_minor)) = (parts.next(), parts.next()) {
+ if major >= 12 {
+ return;
+ }
+ }
+ panic!("\n\nbad LLVM version: {}, need >=12.0\n\n", version)
+}
+
+fn configure_cmake(
+ builder: &Builder<'_>,
+ target: TargetSelection,
+ cfg: &mut cmake::Config,
+ use_compiler_launcher: bool,
+ mut ldflags: LdFlags,
+) {
+ // Do not print installation messages for up-to-date files.
+ // LLVM and LLD builds can produce a lot of those and hit CI limits on log size.
+ cfg.define("CMAKE_INSTALL_MESSAGE", "LAZY");
+
+ // Do not allow the user's value of DESTDIR to influence where
+ // LLVM will install itself. LLVM must always be installed in our
+ // own build directories.
+ cfg.env("DESTDIR", "");
+
+ if builder.ninja() {
+ cfg.generator("Ninja");
+ }
+ cfg.target(&target.triple).host(&builder.config.build.triple);
+
+ if target != builder.config.build {
+ cfg.define("CMAKE_CROSSCOMPILING", "True");
+
+ if target.contains("netbsd") {
+ cfg.define("CMAKE_SYSTEM_NAME", "NetBSD");
+ } else if target.contains("freebsd") {
+ cfg.define("CMAKE_SYSTEM_NAME", "FreeBSD");
+ } else if target.contains("windows") {
+ cfg.define("CMAKE_SYSTEM_NAME", "Windows");
+ } else if target.contains("haiku") {
+ cfg.define("CMAKE_SYSTEM_NAME", "Haiku");
+ } else if target.contains("solaris") || target.contains("illumos") {
+ cfg.define("CMAKE_SYSTEM_NAME", "SunOS");
+ }
+ // When cross-compiling we should also set CMAKE_SYSTEM_VERSION, but in
+ // that case like CMake we cannot easily determine system version either.
+ //
+ // Since, the LLVM itself makes rather limited use of version checks in
+ // CMakeFiles (and then only in tests), and so far no issues have been
+ // reported, the system version is currently left unset.
+
+ if target.contains("darwin") {
+ // Make sure that CMake does not build universal binaries on macOS.
+ // Explicitly specifiy the one single target architecture.
+ if target.starts_with("aarch64") {
+ // macOS uses a different name for building arm64
+ cfg.define("CMAKE_OSX_ARCHITECTURES", "arm64");
+ } else {
+ cfg.define("CMAKE_OSX_ARCHITECTURES", target.triple.split('-').next().unwrap());
+ }
+ }
+ }
+
+ let sanitize_cc = |cc: &Path| {
+ if target.contains("msvc") {
+ OsString::from(cc.to_str().unwrap().replace("\\", "/"))
+ } else {
+ cc.as_os_str().to_owned()
+ }
+ };
+
+ // MSVC with CMake uses msbuild by default which doesn't respect these
+ // vars that we'd otherwise configure. In that case we just skip this
+ // entirely.
+ if target.contains("msvc") && !builder.ninja() {
+ return;
+ }
+
+ let (cc, cxx) = match builder.config.llvm_clang_cl {
+ Some(ref cl) => (cl.as_ref(), cl.as_ref()),
+ None => (builder.cc(target), builder.cxx(target).unwrap()),
+ };
+
+ // Handle msvc + ninja + ccache specially (this is what the bots use)
+ if target.contains("msvc") && builder.ninja() && builder.config.ccache.is_some() {
+ let mut wrap_cc = env::current_exe().expect("failed to get cwd");
+ wrap_cc.set_file_name("sccache-plus-cl.exe");
+
+ cfg.define("CMAKE_C_COMPILER", sanitize_cc(&wrap_cc))
+ .define("CMAKE_CXX_COMPILER", sanitize_cc(&wrap_cc));
+ cfg.env("SCCACHE_PATH", builder.config.ccache.as_ref().unwrap())
+ .env("SCCACHE_TARGET", target.triple)
+ .env("SCCACHE_CC", &cc)
+ .env("SCCACHE_CXX", &cxx);
+
+ // Building LLVM on MSVC can be a little ludicrous at times. We're so far
+ // off the beaten path here that I'm not really sure this is even half
+ // supported any more. Here we're trying to:
+ //
+ // * Build LLVM on MSVC
+ // * Build LLVM with `clang-cl` instead of `cl.exe`
+ // * Build a project with `sccache`
+ // * Build for 32-bit as well
+ // * Build with Ninja
+ //
+ // For `cl.exe` there are different binaries to compile 32/64 bit which
+ // we use but for `clang-cl` there's only one which internally
+ // multiplexes via flags. As a result it appears that CMake's detection
+ // of a compiler's architecture and such on MSVC **doesn't** pass any
+ // custom flags we pass in CMAKE_CXX_FLAGS below. This means that if we
+ // use `clang-cl.exe` it's always diagnosed as a 64-bit compiler which
+ // definitely causes problems since all the env vars are pointing to
+ // 32-bit libraries.
+ //
+ // To hack around this... again... we pass an argument that's
+ // unconditionally passed in the sccache shim. This'll get CMake to
+ // correctly diagnose it's doing a 32-bit compilation and LLVM will
+ // internally configure itself appropriately.
+ if builder.config.llvm_clang_cl.is_some() && target.contains("i686") {
+ cfg.env("SCCACHE_EXTRA_ARGS", "-m32");
+ }
+ } else {
+ // If ccache is configured we inform the build a little differently how
+ // to invoke ccache while also invoking our compilers.
+ if use_compiler_launcher {
+ if let Some(ref ccache) = builder.config.ccache {
+ cfg.define("CMAKE_C_COMPILER_LAUNCHER", ccache)
+ .define("CMAKE_CXX_COMPILER_LAUNCHER", ccache);
+ }
+ }
+ cfg.define("CMAKE_C_COMPILER", sanitize_cc(cc))
+ .define("CMAKE_CXX_COMPILER", sanitize_cc(cxx))
+ .define("CMAKE_ASM_COMPILER", sanitize_cc(cc));
+ }
+
+ cfg.build_arg("-j").build_arg(builder.jobs().to_string());
+ let mut cflags: OsString = builder.cflags(target, GitRepo::Llvm, CLang::C).join(" ").into();
+ if let Some(ref s) = builder.config.llvm_cflags {
+ cflags.push(" ");
+ cflags.push(s);
+ }
+ // Some compiler features used by LLVM (such as thread locals) will not work on a min version below iOS 10.
+ if target.contains("apple-ios") {
+ if target.contains("86-") {
+ cflags.push(" -miphonesimulator-version-min=10.0");
+ } else {
+ cflags.push(" -miphoneos-version-min=10.0");
+ }
+ }
+ if builder.config.llvm_clang_cl.is_some() {
+ cflags.push(&format!(" --target={}", target));
+ }
+ cfg.define("CMAKE_C_FLAGS", cflags);
+ let mut cxxflags: OsString = builder.cflags(target, GitRepo::Llvm, CLang::Cxx).join(" ").into();
+ if let Some(ref s) = builder.config.llvm_cxxflags {
+ cxxflags.push(" ");
+ cxxflags.push(s);
+ }
+ if builder.config.llvm_clang_cl.is_some() {
+ cxxflags.push(&format!(" --target={}", target));
+ }
+ cfg.define("CMAKE_CXX_FLAGS", cxxflags);
+ if let Some(ar) = builder.ar(target) {
+ if ar.is_absolute() {
+ // LLVM build breaks if `CMAKE_AR` is a relative path, for some reason it
+ // tries to resolve this path in the LLVM build directory.
+ cfg.define("CMAKE_AR", sanitize_cc(ar));
+ }
+ }
+
+ if let Some(ranlib) = builder.ranlib(target) {
+ if ranlib.is_absolute() {
+ // LLVM build breaks if `CMAKE_RANLIB` is a relative path, for some reason it
+ // tries to resolve this path in the LLVM build directory.
+ cfg.define("CMAKE_RANLIB", sanitize_cc(ranlib));
+ }
+ }
+
+ if let Some(ref flags) = builder.config.llvm_ldflags {
+ ldflags.push_all(flags);
+ }
+
+ if let Some(flags) = get_var("LDFLAGS", &builder.config.build.triple, &target.triple) {
+ ldflags.push_all(&flags);
+ }
+
+ // For distribution we want the LLVM tools to be *statically* linked to libstdc++.
+ // We also do this if the user explicitly requested static libstdc++.
+ if builder.config.llvm_static_stdcpp {
+ if !target.contains("msvc") && !target.contains("netbsd") && !target.contains("solaris") {
+ if target.contains("apple") || target.contains("windows") {
+ ldflags.push_all("-static-libstdc++");
+ } else {
+ ldflags.push_all("-Wl,-Bsymbolic -static-libstdc++");
+ }
+ }
+ }
+
+ cfg.define("CMAKE_SHARED_LINKER_FLAGS", &ldflags.shared);
+ cfg.define("CMAKE_MODULE_LINKER_FLAGS", &ldflags.module);
+ cfg.define("CMAKE_EXE_LINKER_FLAGS", &ldflags.exe);
+
+ if env::var_os("SCCACHE_ERROR_LOG").is_some() {
+ cfg.env("RUSTC_LOG", "sccache=warn");
+ }
+}
+
+fn configure_llvm(builder: &Builder<'_>, target: TargetSelection, cfg: &mut cmake::Config) {
+ // ThinLTO is only available when building with LLVM, enabling LLD is required.
+ // Apple's linker ld64 supports ThinLTO out of the box though, so don't use LLD on Darwin.
+ if builder.config.llvm_thin_lto {
+ cfg.define("LLVM_ENABLE_LTO", "Thin");
+ if !target.contains("apple") {
+ cfg.define("LLVM_ENABLE_LLD", "ON");
+ }
+ }
+
+ if let Some(ref linker) = builder.config.llvm_use_linker {
+ cfg.define("LLVM_USE_LINKER", linker);
+ }
+
+ if builder.config.llvm_allow_old_toolchain {
+ cfg.define("LLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN", "YES");
+ }
+}
+
+// Adapted from https://github.com/alexcrichton/cc-rs/blob/fba7feded71ee4f63cfe885673ead6d7b4f2f454/src/lib.rs#L2347-L2365
+fn get_var(var_base: &str, host: &str, target: &str) -> Option<OsString> {
+ let kind = if host == target { "HOST" } else { "TARGET" };
+ let target_u = target.replace("-", "_");
+ env::var_os(&format!("{}_{}", var_base, target))
+ .or_else(|| env::var_os(&format!("{}_{}", var_base, target_u)))
+ .or_else(|| env::var_os(&format!("{}_{}", kind, var_base)))
+ .or_else(|| env::var_os(var_base))
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Lld {
+ pub target: TargetSelection,
+}
+
+impl Step for Lld {
+ type Output = PathBuf;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/llvm-project/lld")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Lld { target: run.target });
+ }
+
+ /// Compile LLD for `target`.
+ fn run(self, builder: &Builder<'_>) -> PathBuf {
+ if builder.config.dry_run {
+ return PathBuf::from("lld-out-dir-test-gen");
+ }
+ let target = self.target;
+
+ let llvm_config = builder.ensure(Llvm { target: self.target });
+
+ let out_dir = builder.lld_out(target);
+ let done_stamp = out_dir.join("lld-finished-building");
+ if done_stamp.exists() {
+ return out_dir;
+ }
+
+ builder.info(&format!("Building LLD for {}", target));
+ let _time = util::timeit(&builder);
+ t!(fs::create_dir_all(&out_dir));
+
+ let mut cfg = cmake::Config::new(builder.src.join("src/llvm-project/lld"));
+ let mut ldflags = LdFlags::default();
+
+ // When building LLD as part of a build with instrumentation on windows, for example
+ // when doing PGO on CI, cmake or clang-cl don't automatically link clang's
+ // profiler runtime in. In that case, we need to manually ask cmake to do it, to avoid
+ // linking errors, much like LLVM's cmake setup does in that situation.
+ if builder.config.llvm_profile_generate && target.contains("msvc") {
+ if let Some(clang_cl_path) = builder.config.llvm_clang_cl.as_ref() {
+ // Find clang's runtime library directory and push that as a search path to the
+ // cmake linker flags.
+ let clang_rt_dir = get_clang_cl_resource_dir(clang_cl_path);
+ ldflags.push_all(&format!("/libpath:{}", clang_rt_dir.display()));
+ }
+ }
+
+ configure_cmake(builder, target, &mut cfg, true, ldflags);
+ configure_llvm(builder, target, &mut cfg);
+
+ // This is an awful, awful hack. Discovered when we migrated to using
+ // clang-cl to compile LLVM/LLD it turns out that LLD, when built out of
+ // tree, will execute `llvm-config --cmakedir` and then tell CMake about
+ // that directory for later processing. Unfortunately if this path has
+ // forward slashes in it (which it basically always does on Windows)
+ // then CMake will hit a syntax error later on as... something isn't
+ // escaped it seems?
+ //
+ // Instead of attempting to fix this problem in upstream CMake and/or
+ // LLVM/LLD we just hack around it here. This thin wrapper will take the
+ // output from llvm-config and replace all instances of `\` with `/` to
+ // ensure we don't hit the same bugs with escaping. It means that you
+ // can't build on a system where your paths require `\` on Windows, but
+ // there's probably a lot of reasons you can't do that other than this.
+ let llvm_config_shim = env::current_exe().unwrap().with_file_name("llvm-config-wrapper");
+
+ // Re-use the same flags as llvm to control the level of debug information
+ // generated for lld.
+ let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) {
+ (false, _) => "Debug",
+ (true, false) => "Release",
+ (true, true) => "RelWithDebInfo",
+ };
+
+ cfg.out_dir(&out_dir)
+ .profile(profile)
+ .env("LLVM_CONFIG_REAL", &llvm_config)
+ .define("LLVM_CONFIG_PATH", llvm_config_shim)
+ .define("LLVM_INCLUDE_TESTS", "OFF");
+
+ // While we're using this horrible workaround to shim the execution of
+ // llvm-config, let's just pile on more. I can't seem to figure out how
+ // to build LLD as a standalone project and also cross-compile it at the
+ // same time. It wants a natively executable `llvm-config` to learn
+ // about LLVM, but then it learns about all the host configuration of
+ // LLVM and tries to link to host LLVM libraries.
+ //
+ // To work around that we tell our shim to replace anything with the
+ // build target with the actual target instead. This'll break parts of
+ // LLD though which try to execute host tools, such as llvm-tblgen, so
+ // we specifically tell it where to find those. This is likely super
+ // brittle and will break over time. If anyone knows better how to
+ // cross-compile LLD it would be much appreciated to fix this!
+ if target != builder.config.build {
+ cfg.env("LLVM_CONFIG_SHIM_REPLACE", &builder.config.build.triple)
+ .env("LLVM_CONFIG_SHIM_REPLACE_WITH", &target.triple)
+ .define(
+ "LLVM_TABLEGEN_EXE",
+ llvm_config.with_file_name("llvm-tblgen").with_extension(EXE_EXTENSION),
+ );
+ }
+
+ // Explicitly set C++ standard, because upstream doesn't do so
+ // for standalone builds.
+ cfg.define("CMAKE_CXX_STANDARD", "14");
+
+ cfg.build();
+
+ t!(File::create(&done_stamp));
+ out_dir
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct TestHelpers {
+ pub target: TargetSelection,
+}
+
+impl Step for TestHelpers {
+ type Output = ();
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/test/auxiliary/rust_test_helpers.c")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(TestHelpers { target: run.target })
+ }
+
+ /// Compiles the `rust_test_helpers.c` library which we used in various
+ /// `run-pass` tests for ABI testing.
+ fn run(self, builder: &Builder<'_>) {
+ if builder.config.dry_run {
+ return;
+ }
+ // The x86_64-fortanix-unknown-sgx target doesn't have a working C
+ // toolchain. However, some x86_64 ELF objects can be linked
+ // without issues. Use this hack to compile the test helpers.
+ let target = if self.target == "x86_64-fortanix-unknown-sgx" {
+ TargetSelection::from_user("x86_64-unknown-linux-gnu")
+ } else {
+ self.target
+ };
+ let dst = builder.test_helpers_out(target);
+ let src = builder.src.join("src/test/auxiliary/rust_test_helpers.c");
+ if up_to_date(&src, &dst.join("librust_test_helpers.a")) {
+ return;
+ }
+
+ builder.info("Building test helpers");
+ t!(fs::create_dir_all(&dst));
+ let mut cfg = cc::Build::new();
+ // FIXME: Workaround for https://github.com/emscripten-core/emscripten/issues/9013
+ if target.contains("emscripten") {
+ cfg.pic(false);
+ }
+
+ // We may have found various cross-compilers a little differently due to our
+ // extra configuration, so inform cc of these compilers. Note, though, that
+ // on MSVC we still need cc's detection of env vars (ugh).
+ if !target.contains("msvc") {
+ if let Some(ar) = builder.ar(target) {
+ cfg.archiver(ar);
+ }
+ cfg.compiler(builder.cc(target));
+ }
+ cfg.cargo_metadata(false)
+ .out_dir(&dst)
+ .target(&target.triple)
+ .host(&builder.config.build.triple)
+ .opt_level(0)
+ .warnings(false)
+ .debug(false)
+ .file(builder.src.join("src/test/auxiliary/rust_test_helpers.c"))
+ .compile("rust_test_helpers");
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct Sanitizers {
+ pub target: TargetSelection,
+}
+
+impl Step for Sanitizers {
+ type Output = Vec<SanitizerRuntime>;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.alias("sanitizers")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Sanitizers { target: run.target });
+ }
+
+ /// Builds sanitizer runtime libraries.
+ fn run(self, builder: &Builder<'_>) -> Self::Output {
+ let compiler_rt_dir = builder.src.join("src/llvm-project/compiler-rt");
+ if !compiler_rt_dir.exists() {
+ return Vec::new();
+ }
+
+ let out_dir = builder.native_dir(self.target).join("sanitizers");
+ let runtimes = supported_sanitizers(&out_dir, self.target, &builder.config.channel);
+ if runtimes.is_empty() {
+ return runtimes;
+ }
+
+ let llvm_config = builder.ensure(Llvm { target: builder.config.build });
+ if builder.config.dry_run {
+ return runtimes;
+ }
+
+ let stamp = out_dir.join("sanitizers-finished-building");
+ let stamp = HashStamp::new(stamp, builder.in_tree_llvm_info.sha());
+
+ if stamp.is_done() {
+ if stamp.hash.is_none() {
+ builder.info(&format!(
+ "Rebuild sanitizers by removing the file `{}`",
+ stamp.path.display()
+ ));
+ }
+ return runtimes;
+ }
+
+ builder.info(&format!("Building sanitizers for {}", self.target));
+ t!(stamp.remove());
+ let _time = util::timeit(&builder);
+
+ let mut cfg = cmake::Config::new(&compiler_rt_dir);
+ cfg.profile("Release");
+ cfg.define("CMAKE_C_COMPILER_TARGET", self.target.triple);
+ cfg.define("COMPILER_RT_BUILD_BUILTINS", "OFF");
+ cfg.define("COMPILER_RT_BUILD_CRT", "OFF");
+ cfg.define("COMPILER_RT_BUILD_LIBFUZZER", "OFF");
+ cfg.define("COMPILER_RT_BUILD_PROFILE", "OFF");
+ cfg.define("COMPILER_RT_BUILD_SANITIZERS", "ON");
+ cfg.define("COMPILER_RT_BUILD_XRAY", "OFF");
+ cfg.define("COMPILER_RT_DEFAULT_TARGET_ONLY", "ON");
+ cfg.define("COMPILER_RT_USE_LIBCXX", "OFF");
+ cfg.define("LLVM_CONFIG_PATH", &llvm_config);
+
+ // On Darwin targets the sanitizer runtimes are build as universal binaries.
+ // Unfortunately sccache currently lacks support to build them successfully.
+ // Disable compiler launcher on Darwin targets to avoid potential issues.
+ let use_compiler_launcher = !self.target.contains("apple-darwin");
+ configure_cmake(builder, self.target, &mut cfg, use_compiler_launcher, LdFlags::default());
+
+ t!(fs::create_dir_all(&out_dir));
+ cfg.out_dir(out_dir);
+
+ for runtime in &runtimes {
+ cfg.build_target(&runtime.cmake_target);
+ cfg.build();
+ }
+ t!(stamp.write());
+
+ runtimes
+ }
+}
+
+#[derive(Clone, Debug)]
+pub struct SanitizerRuntime {
+ /// CMake target used to build the runtime.
+ pub cmake_target: String,
+ /// Path to the built runtime library.
+ pub path: PathBuf,
+ /// Library filename that will be used rustc.
+ pub name: String,
+}
+
+/// Returns sanitizers available on a given target.
+fn supported_sanitizers(
+ out_dir: &Path,
+ target: TargetSelection,
+ channel: &str,
+) -> Vec<SanitizerRuntime> {
+ let darwin_libs = |os: &str, components: &[&str]| -> Vec<SanitizerRuntime> {
+ components
+ .iter()
+ .map(move |c| SanitizerRuntime {
+ cmake_target: format!("clang_rt.{}_{}_dynamic", c, os),
+ path: out_dir
+ .join(&format!("build/lib/darwin/libclang_rt.{}_{}_dynamic.dylib", c, os)),
+ name: format!("librustc-{}_rt.{}.dylib", channel, c),
+ })
+ .collect()
+ };
+
+ let common_libs = |os: &str, arch: &str, components: &[&str]| -> Vec<SanitizerRuntime> {
+ components
+ .iter()
+ .map(move |c| SanitizerRuntime {
+ cmake_target: format!("clang_rt.{}-{}", c, arch),
+ path: out_dir.join(&format!("build/lib/{}/libclang_rt.{}-{}.a", os, c, arch)),
+ name: format!("librustc-{}_rt.{}.a", channel, c),
+ })
+ .collect()
+ };
+
+ match &*target.triple {
+ "aarch64-apple-darwin" => darwin_libs("osx", &["asan", "lsan", "tsan"]),
+ "aarch64-fuchsia" => common_libs("fuchsia", "aarch64", &["asan"]),
+ "aarch64-unknown-linux-gnu" => {
+ common_libs("linux", "aarch64", &["asan", "lsan", "msan", "tsan", "hwasan"])
+ }
+ "x86_64-apple-darwin" => darwin_libs("osx", &["asan", "lsan", "tsan"]),
+ "x86_64-fuchsia" => common_libs("fuchsia", "x86_64", &["asan"]),
+ "x86_64-unknown-freebsd" => common_libs("freebsd", "x86_64", &["asan", "msan", "tsan"]),
+ "x86_64-unknown-netbsd" => {
+ common_libs("netbsd", "x86_64", &["asan", "lsan", "msan", "tsan"])
+ }
+ "x86_64-unknown-illumos" => common_libs("illumos", "x86_64", &["asan"]),
+ "x86_64-pc-solaris" => common_libs("solaris", "x86_64", &["asan"]),
+ "x86_64-unknown-linux-gnu" => {
+ common_libs("linux", "x86_64", &["asan", "lsan", "msan", "tsan"])
+ }
+ "x86_64-unknown-linux-musl" => {
+ common_libs("linux", "x86_64", &["asan", "lsan", "msan", "tsan"])
+ }
+ _ => Vec::new(),
+ }
+}
+
+struct HashStamp {
+ path: PathBuf,
+ hash: Option<Vec<u8>>,
+}
+
+impl HashStamp {
+ fn new(path: PathBuf, hash: Option<&str>) -> Self {
+ HashStamp { path, hash: hash.map(|s| s.as_bytes().to_owned()) }
+ }
+
+ fn is_done(&self) -> bool {
+ match fs::read(&self.path) {
+ Ok(h) => self.hash.as_deref().unwrap_or(b"") == h.as_slice(),
+ Err(e) if e.kind() == io::ErrorKind::NotFound => false,
+ Err(e) => {
+ panic!("failed to read stamp file `{}`: {}", self.path.display(), e);
+ }
+ }
+ }
+
+ fn remove(&self) -> io::Result<()> {
+ match fs::remove_file(&self.path) {
+ Ok(()) => Ok(()),
+ Err(e) => {
+ if e.kind() == io::ErrorKind::NotFound {
+ Ok(())
+ } else {
+ Err(e)
+ }
+ }
+ }
+ }
+
+ fn write(&self) -> io::Result<()> {
+ fs::write(&self.path, self.hash.as_deref().unwrap_or(b""))
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct CrtBeginEnd {
+ pub target: TargetSelection,
+}
+
+impl Step for CrtBeginEnd {
+ type Output = PathBuf;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/llvm-project/compiler-rt/lib/crt")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(CrtBeginEnd { target: run.target });
+ }
+
+ /// Build crtbegin.o/crtend.o for musl target.
+ fn run(self, builder: &Builder<'_>) -> Self::Output {
+ let out_dir = builder.native_dir(self.target).join("crt");
+
+ if builder.config.dry_run {
+ return out_dir;
+ }
+
+ let crtbegin_src = builder.src.join("src/llvm-project/compiler-rt/lib/crt/crtbegin.c");
+ let crtend_src = builder.src.join("src/llvm-project/compiler-rt/lib/crt/crtend.c");
+ if up_to_date(&crtbegin_src, &out_dir.join("crtbegin.o"))
+ && up_to_date(&crtend_src, &out_dir.join("crtendS.o"))
+ {
+ return out_dir;
+ }
+
+ builder.info("Building crtbegin.o and crtend.o");
+ t!(fs::create_dir_all(&out_dir));
+
+ let mut cfg = cc::Build::new();
+
+ if let Some(ar) = builder.ar(self.target) {
+ cfg.archiver(ar);
+ }
+ cfg.compiler(builder.cc(self.target));
+ cfg.cargo_metadata(false)
+ .out_dir(&out_dir)
+ .target(&self.target.triple)
+ .host(&builder.config.build.triple)
+ .warnings(false)
+ .debug(false)
+ .opt_level(3)
+ .file(crtbegin_src)
+ .file(crtend_src);
+
+ // Those flags are defined in src/llvm-project/compiler-rt/lib/crt/CMakeLists.txt
+ // Currently only consumer of those objects is musl, which use .init_array/.fini_array
+ // instead of .ctors/.dtors
+ cfg.flag("-std=c11")
+ .define("CRT_HAS_INITFINI_ARRAY", None)
+ .define("EH_USE_FRAME_REGISTRY", None);
+
+ cfg.compile("crt");
+
+ t!(fs::copy(out_dir.join("crtbegin.o"), out_dir.join("crtbeginS.o")));
+ t!(fs::copy(out_dir.join("crtend.o"), out_dir.join("crtendS.o")));
+ out_dir
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct Libunwind {
+ pub target: TargetSelection,
+}
+
+impl Step for Libunwind {
+ type Output = PathBuf;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/llvm-project/libunwind")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Libunwind { target: run.target });
+ }
+
+ /// Build linunwind.a
+ fn run(self, builder: &Builder<'_>) -> Self::Output {
+ if builder.config.dry_run {
+ return PathBuf::new();
+ }
+
+ let out_dir = builder.native_dir(self.target).join("libunwind");
+ let root = builder.src.join("src/llvm-project/libunwind");
+
+ if up_to_date(&root, &out_dir.join("libunwind.a")) {
+ return out_dir;
+ }
+
+ builder.info(&format!("Building libunwind.a for {}", self.target.triple));
+ t!(fs::create_dir_all(&out_dir));
+
+ let mut cc_cfg = cc::Build::new();
+ let mut cpp_cfg = cc::Build::new();
+
+ cpp_cfg.cpp(true);
+ cpp_cfg.cpp_set_stdlib(None);
+ cpp_cfg.flag("-nostdinc++");
+ cpp_cfg.flag("-fno-exceptions");
+ cpp_cfg.flag("-fno-rtti");
+ cpp_cfg.flag_if_supported("-fvisibility-global-new-delete-hidden");
+
+ for cfg in [&mut cc_cfg, &mut cpp_cfg].iter_mut() {
+ if let Some(ar) = builder.ar(self.target) {
+ cfg.archiver(ar);
+ }
+ cfg.target(&self.target.triple);
+ cfg.host(&builder.config.build.triple);
+ cfg.warnings(false);
+ cfg.debug(false);
+ // get_compiler() need set opt_level first.
+ cfg.opt_level(3);
+ cfg.flag("-fstrict-aliasing");
+ cfg.flag("-funwind-tables");
+ cfg.flag("-fvisibility=hidden");
+ cfg.define("_LIBUNWIND_DISABLE_VISIBILITY_ANNOTATIONS", None);
+ cfg.include(root.join("include"));
+ cfg.cargo_metadata(false);
+ cfg.out_dir(&out_dir);
+
+ if self.target.contains("x86_64-fortanix-unknown-sgx") {
+ cfg.static_flag(true);
+ cfg.flag("-fno-stack-protector");
+ cfg.flag("-ffreestanding");
+ cfg.flag("-fexceptions");
+
+ // easiest way to undefine since no API available in cc::Build to undefine
+ cfg.flag("-U_FORTIFY_SOURCE");
+ cfg.define("_FORTIFY_SOURCE", "0");
+ cfg.define("RUST_SGX", "1");
+ cfg.define("__NO_STRING_INLINES", None);
+ cfg.define("__NO_MATH_INLINES", None);
+ cfg.define("_LIBUNWIND_IS_BAREMETAL", None);
+ cfg.define("__LIBUNWIND_IS_NATIVE_ONLY", None);
+ cfg.define("NDEBUG", None);
+ }
+ if self.target.contains("windows") {
+ cfg.define("_LIBUNWIND_HIDE_SYMBOLS", "1");
+ cfg.define("_LIBUNWIND_IS_NATIVE_ONLY", "1");
+ }
+ }
+
+ cc_cfg.compiler(builder.cc(self.target));
+ if let Ok(cxx) = builder.cxx(self.target) {
+ cpp_cfg.compiler(cxx);
+ } else {
+ cc_cfg.compiler(builder.cc(self.target));
+ }
+
+ // Don't set this for clang
+ // By default, Clang builds C code in GNU C17 mode.
+ // By default, Clang builds C++ code according to the C++98 standard,
+ // with many C++11 features accepted as extensions.
+ if cc_cfg.get_compiler().is_like_gnu() {
+ cc_cfg.flag("-std=c99");
+ }
+ if cpp_cfg.get_compiler().is_like_gnu() {
+ cpp_cfg.flag("-std=c++11");
+ }
+
+ if self.target.contains("x86_64-fortanix-unknown-sgx") || self.target.contains("musl") {
+ // use the same GCC C compiler command to compile C++ code so we do not need to setup the
+ // C++ compiler env variables on the builders.
+ // Don't set this for clang++, as clang++ is able to compile this without libc++.
+ if cpp_cfg.get_compiler().is_like_gnu() {
+ cpp_cfg.cpp(false);
+ cpp_cfg.compiler(builder.cc(self.target));
+ }
+ }
+
+ let mut c_sources = vec![
+ "Unwind-sjlj.c",
+ "UnwindLevel1-gcc-ext.c",
+ "UnwindLevel1.c",
+ "UnwindRegistersRestore.S",
+ "UnwindRegistersSave.S",
+ ];
+
+ let cpp_sources = vec!["Unwind-EHABI.cpp", "Unwind-seh.cpp", "libunwind.cpp"];
+ let cpp_len = cpp_sources.len();
+
+ if self.target.contains("x86_64-fortanix-unknown-sgx") {
+ c_sources.push("UnwindRustSgx.c");
+ }
+
+ for src in c_sources {
+ cc_cfg.file(root.join("src").join(src).canonicalize().unwrap());
+ }
+
+ for src in &cpp_sources {
+ cpp_cfg.file(root.join("src").join(src).canonicalize().unwrap());
+ }
+
+ cpp_cfg.compile("unwind-cpp");
+
+ // FIXME: https://github.com/alexcrichton/cc-rs/issues/545#issuecomment-679242845
+ let mut count = 0;
+ for entry in fs::read_dir(&out_dir).unwrap() {
+ let file = entry.unwrap().path().canonicalize().unwrap();
+ if file.is_file() && file.extension() == Some(OsStr::new("o")) {
+ // file name starts with "Unwind-EHABI", "Unwind-seh" or "libunwind"
+ let file_name = file.file_name().unwrap().to_str().expect("UTF-8 file name");
+ if cpp_sources.iter().any(|f| file_name.starts_with(&f[..f.len() - 4])) {
+ cc_cfg.object(&file);
+ count += 1;
+ }
+ }
+ }
+ assert_eq!(cpp_len, count, "Can't get object files from {:?}", &out_dir);
+
+ cc_cfg.compile("unwind");
+ out_dir
+ }
+}
diff --git a/src/bootstrap/run.rs b/src/bootstrap/run.rs
new file mode 100644
index 000000000..25abe7a72
--- /dev/null
+++ b/src/bootstrap/run.rs
@@ -0,0 +1,105 @@
+use crate::builder::{Builder, RunConfig, ShouldRun, Step};
+use crate::dist::distdir;
+use crate::tool::Tool;
+use crate::util::output;
+use std::process::Command;
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct ExpandYamlAnchors;
+
+impl Step for ExpandYamlAnchors {
+ type Output = ();
+
+ /// Runs the `expand-yaml_anchors` tool.
+ ///
+ /// This tool in `src/tools` reads the CI configuration files written in YAML and expands the
+ /// anchors in them, since GitHub Actions doesn't support them.
+ fn run(self, builder: &Builder<'_>) {
+ builder.info("Expanding YAML anchors in the GitHub Actions configuration");
+ try_run(
+ builder,
+ &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("generate").arg(&builder.src),
+ );
+ }
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/tools/expand-yaml-anchors")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(ExpandYamlAnchors);
+ }
+}
+
+fn try_run(builder: &Builder<'_>, cmd: &mut Command) -> bool {
+ if !builder.fail_fast {
+ if !builder.try_run(cmd) {
+ let mut failures = builder.delayed_failures.borrow_mut();
+ failures.push(format!("{:?}", cmd));
+ return false;
+ }
+ } else {
+ builder.run(cmd);
+ }
+ true
+}
+
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct BuildManifest;
+
+impl Step for BuildManifest {
+ type Output = ();
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/tools/build-manifest")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(BuildManifest);
+ }
+
+ fn run(self, builder: &Builder<'_>) {
+ // This gets called by `promote-release`
+ // (https://github.com/rust-lang/promote-release).
+ let mut cmd = builder.tool_cmd(Tool::BuildManifest);
+ let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
+ panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
+ });
+ let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
+ panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
+ });
+
+ let today = output(Command::new("date").arg("+%Y-%m-%d"));
+
+ cmd.arg(sign);
+ cmd.arg(distdir(builder));
+ cmd.arg(today.trim());
+ cmd.arg(addr);
+ cmd.arg(&builder.config.channel);
+
+ builder.create_dir(&distdir(builder));
+ builder.run(&mut cmd);
+ }
+}
+
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct BumpStage0;
+
+impl Step for BumpStage0 {
+ type Output = ();
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/tools/bump-stage0")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(BumpStage0);
+ }
+
+ fn run(self, builder: &Builder<'_>) -> Self::Output {
+ let mut cmd = builder.tool_cmd(Tool::BumpStage0);
+ builder.run(&mut cmd);
+ }
+}
diff --git a/src/bootstrap/sanity.rs b/src/bootstrap/sanity.rs
new file mode 100644
index 000000000..cae41286f
--- /dev/null
+++ b/src/bootstrap/sanity.rs
@@ -0,0 +1,242 @@
+//! Sanity checking performed by rustbuild before actually executing anything.
+//!
+//! This module contains the implementation of ensuring that the build
+//! environment looks reasonable before progressing. This will verify that
+//! various programs like git and python exist, along with ensuring that all C
+//! compilers for cross-compiling are found.
+//!
+//! In theory if we get past this phase it's a bug if a build fails, but in
+//! practice that's likely not true!
+
+use std::collections::HashMap;
+use std::env;
+use std::ffi::{OsStr, OsString};
+use std::fs;
+use std::path::PathBuf;
+use std::process::Command;
+
+use crate::cache::INTERNER;
+use crate::config::Target;
+use crate::util::output;
+use crate::Build;
+
+pub struct Finder {
+ cache: HashMap<OsString, Option<PathBuf>>,
+ path: OsString,
+}
+
+impl Finder {
+ pub fn new() -> Self {
+ Self { cache: HashMap::new(), path: env::var_os("PATH").unwrap_or_default() }
+ }
+
+ pub fn maybe_have<S: Into<OsString>>(&mut self, cmd: S) -> Option<PathBuf> {
+ let cmd: OsString = cmd.into();
+ let path = &self.path;
+ self.cache
+ .entry(cmd.clone())
+ .or_insert_with(|| {
+ for path in env::split_paths(path) {
+ let target = path.join(&cmd);
+ let mut cmd_exe = cmd.clone();
+ cmd_exe.push(".exe");
+
+ if target.is_file() // some/path/git
+ || path.join(&cmd_exe).exists() // some/path/git.exe
+ || target.join(&cmd_exe).exists()
+ // some/path/git/git.exe
+ {
+ return Some(target);
+ }
+ }
+ None
+ })
+ .clone()
+ }
+
+ pub fn must_have<S: AsRef<OsStr>>(&mut self, cmd: S) -> PathBuf {
+ self.maybe_have(&cmd).unwrap_or_else(|| {
+ panic!("\n\ncouldn't find required command: {:?}\n\n", cmd.as_ref());
+ })
+ }
+}
+
+pub fn check(build: &mut Build) {
+ let path = env::var_os("PATH").unwrap_or_default();
+ // On Windows, quotes are invalid characters for filename paths, and if
+ // one is present as part of the PATH then that can lead to the system
+ // being unable to identify the files properly. See
+ // https://github.com/rust-lang/rust/issues/34959 for more details.
+ if cfg!(windows) && path.to_string_lossy().contains('\"') {
+ panic!("PATH contains invalid character '\"'");
+ }
+
+ let mut cmd_finder = Finder::new();
+ // If we've got a git directory we're gonna need git to update
+ // submodules and learn about various other aspects.
+ if build.rust_info.is_git() {
+ cmd_finder.must_have("git");
+ }
+
+ // We need cmake, but only if we're actually building LLVM or sanitizers.
+ let building_llvm = build.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm"))
+ && build
+ .hosts
+ .iter()
+ .map(|host| {
+ build
+ .config
+ .target_config
+ .get(host)
+ .map(|config| config.llvm_config.is_none())
+ .unwrap_or(true)
+ })
+ .any(|build_llvm_ourselves| build_llvm_ourselves);
+ let need_cmake = building_llvm || build.config.any_sanitizers_enabled();
+ if need_cmake {
+ if cmd_finder.maybe_have("cmake").is_none() {
+ eprintln!(
+ "
+Couldn't find required command: cmake
+
+You should install cmake, or set `download-ci-llvm = true` in the
+`[llvm]` section section of `config.toml` to download LLVM rather
+than building it.
+"
+ );
+ crate::detail_exit(1);
+ }
+ }
+
+ build.config.python = build
+ .config
+ .python
+ .take()
+ .map(|p| cmd_finder.must_have(p))
+ .or_else(|| env::var_os("BOOTSTRAP_PYTHON").map(PathBuf::from)) // set by bootstrap.py
+ .or_else(|| cmd_finder.maybe_have("python"))
+ .or_else(|| cmd_finder.maybe_have("python3"))
+ .or_else(|| cmd_finder.maybe_have("python2"));
+
+ build.config.nodejs = build
+ .config
+ .nodejs
+ .take()
+ .map(|p| cmd_finder.must_have(p))
+ .or_else(|| cmd_finder.maybe_have("node"))
+ .or_else(|| cmd_finder.maybe_have("nodejs"));
+
+ build.config.npm = build
+ .config
+ .npm
+ .take()
+ .map(|p| cmd_finder.must_have(p))
+ .or_else(|| cmd_finder.maybe_have("npm"));
+
+ build.config.gdb = build
+ .config
+ .gdb
+ .take()
+ .map(|p| cmd_finder.must_have(p))
+ .or_else(|| cmd_finder.maybe_have("gdb"));
+
+ // We're gonna build some custom C code here and there, host triples
+ // also build some C++ shims for LLVM so we need a C++ compiler.
+ for target in &build.targets {
+ // On emscripten we don't actually need the C compiler to just
+ // build the target artifacts, only for testing. For the sake
+ // of easier bot configuration, just skip detection.
+ if target.contains("emscripten") {
+ continue;
+ }
+
+ // We don't use a C compiler on wasm32
+ if target.contains("wasm32") {
+ continue;
+ }
+
+ if !build.config.dry_run {
+ cmd_finder.must_have(build.cc(*target));
+ if let Some(ar) = build.ar(*target) {
+ cmd_finder.must_have(ar);
+ }
+ }
+ }
+
+ for host in &build.hosts {
+ if !build.config.dry_run {
+ cmd_finder.must_have(build.cxx(*host).unwrap());
+ }
+ }
+
+ if build.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) {
+ // Externally configured LLVM requires FileCheck to exist
+ let filecheck = build.llvm_filecheck(build.build);
+ if !filecheck.starts_with(&build.out) && !filecheck.exists() && build.config.codegen_tests {
+ panic!("FileCheck executable {:?} does not exist", filecheck);
+ }
+ }
+
+ for target in &build.targets {
+ build
+ .config
+ .target_config
+ .entry(*target)
+ .or_insert_with(|| Target::from_triple(&target.triple));
+
+ if target.contains("-none-") || target.contains("nvptx") {
+ if build.no_std(*target) == Some(false) {
+ panic!("All the *-none-* and nvptx* targets are no-std targets")
+ }
+ }
+
+ // Make sure musl-root is valid
+ if target.contains("musl") {
+ // If this is a native target (host is also musl) and no musl-root is given,
+ // fall back to the system toolchain in /usr before giving up
+ if build.musl_root(*target).is_none() && build.config.build == *target {
+ let target = build.config.target_config.entry(*target).or_default();
+ target.musl_root = Some("/usr".into());
+ }
+ match build.musl_libdir(*target) {
+ Some(libdir) => {
+ if fs::metadata(libdir.join("libc.a")).is_err() {
+ panic!("couldn't find libc.a in musl libdir: {}", libdir.display());
+ }
+ }
+ None => panic!(
+ "when targeting MUSL either the rust.musl-root \
+ option or the target.$TARGET.musl-root option must \
+ be specified in config.toml"
+ ),
+ }
+ }
+
+ if need_cmake && target.contains("msvc") {
+ // There are three builds of cmake on windows: MSVC, MinGW, and
+ // Cygwin. The Cygwin build does not have generators for Visual
+ // Studio, so detect that here and error.
+ let out = output(Command::new("cmake").arg("--help"));
+ if !out.contains("Visual Studio") {
+ panic!(
+ "
+cmake does not support Visual Studio generators.
+
+This is likely due to it being an msys/cygwin build of cmake,
+rather than the required windows version, built using MinGW
+or Visual Studio.
+
+If you are building under msys2 try installing the mingw-w64-x86_64-cmake
+package instead of cmake:
+
+$ pacman -R cmake && pacman -S mingw-w64-x86_64-cmake
+"
+ );
+ }
+ }
+ }
+
+ if let Some(ref s) = build.config.ccache {
+ cmd_finder.must_have(s);
+ }
+}
diff --git a/src/bootstrap/setup.rs b/src/bootstrap/setup.rs
new file mode 100644
index 000000000..a5a39a5a3
--- /dev/null
+++ b/src/bootstrap/setup.rs
@@ -0,0 +1,350 @@
+use crate::{t, VERSION};
+use crate::{Config, TargetSelection};
+use std::env::consts::EXE_SUFFIX;
+use std::fmt::Write as _;
+use std::fs::File;
+use std::path::{Path, PathBuf, MAIN_SEPARATOR};
+use std::process::Command;
+use std::str::FromStr;
+use std::{
+ env, fmt, fs,
+ io::{self, Write},
+};
+
+#[derive(Clone, Copy, Eq, PartialEq)]
+pub enum Profile {
+ Compiler,
+ Codegen,
+ Library,
+ Tools,
+ User,
+}
+
+impl Profile {
+ fn include_path(&self, src_path: &Path) -> PathBuf {
+ PathBuf::from(format!("{}/src/bootstrap/defaults/config.{}.toml", src_path.display(), self))
+ }
+
+ pub fn all() -> impl Iterator<Item = Self> {
+ use Profile::*;
+ // N.B. these are ordered by how they are displayed, not alphabetically
+ [Library, Compiler, Codegen, Tools, User].iter().copied()
+ }
+
+ pub fn purpose(&self) -> String {
+ use Profile::*;
+ match self {
+ Library => "Contribute to the standard library",
+ Compiler => "Contribute to the compiler itself",
+ Codegen => "Contribute to the compiler, and also modify LLVM or codegen",
+ Tools => "Contribute to tools which depend on the compiler, but do not modify it directly (e.g. rustdoc, clippy, miri)",
+ User => "Install Rust from source",
+ }
+ .to_string()
+ }
+
+ pub fn all_for_help(indent: &str) -> String {
+ let mut out = String::new();
+ for choice in Profile::all() {
+ writeln!(&mut out, "{}{}: {}", indent, choice, choice.purpose()).unwrap();
+ }
+ out
+ }
+}
+
+impl FromStr for Profile {
+ type Err = String;
+
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
+ match s {
+ "lib" | "library" => Ok(Profile::Library),
+ "compiler" => Ok(Profile::Compiler),
+ "llvm" | "codegen" => Ok(Profile::Codegen),
+ "maintainer" | "user" => Ok(Profile::User),
+ "tools" | "tool" | "rustdoc" | "clippy" | "miri" | "rustfmt" | "rls" => {
+ Ok(Profile::Tools)
+ }
+ _ => Err(format!("unknown profile: '{}'", s)),
+ }
+ }
+}
+
+impl fmt::Display for Profile {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ match self {
+ Profile::Compiler => write!(f, "compiler"),
+ Profile::Codegen => write!(f, "codegen"),
+ Profile::Library => write!(f, "library"),
+ Profile::User => write!(f, "user"),
+ Profile::Tools => write!(f, "tools"),
+ }
+ }
+}
+
+pub fn setup(config: &Config, profile: Profile) {
+ let path = &config.config;
+
+ if path.exists() {
+ eprintln!(
+ "error: you asked `x.py` to setup a new config file, but one already exists at `{}`",
+ path.display()
+ );
+ eprintln!("help: try adding `profile = \"{}\"` at the top of {}", profile, path.display());
+ eprintln!(
+ "note: this will use the configuration in {}",
+ profile.include_path(&config.src).display()
+ );
+ crate::detail_exit(1);
+ }
+
+ let settings = format!(
+ "# Includes one of the default files in src/bootstrap/defaults\n\
+ profile = \"{}\"\n\
+ changelog-seen = {}\n",
+ profile, VERSION
+ );
+ t!(fs::write(path, settings));
+
+ let include_path = profile.include_path(&config.src);
+ println!("`x.py` will now use the configuration at {}", include_path.display());
+
+ let build = TargetSelection::from_user(&env!("BUILD_TRIPLE"));
+ let stage_path =
+ ["build", build.rustc_target_arg(), "stage1"].join(&MAIN_SEPARATOR.to_string());
+
+ println!();
+
+ if !rustup_installed() && profile != Profile::User {
+ eprintln!("`rustup` is not installed; cannot link `stage1` toolchain");
+ } else if stage_dir_exists(&stage_path[..]) {
+ attempt_toolchain_link(&stage_path[..]);
+ }
+
+ let suggestions = match profile {
+ Profile::Codegen | Profile::Compiler => &["check", "build", "test"][..],
+ Profile::Tools => &[
+ "check",
+ "build",
+ "test src/test/rustdoc*",
+ "test src/tools/clippy",
+ "test src/tools/miri",
+ "test src/tools/rustfmt",
+ ],
+ Profile::Library => &["check", "build", "test library/std", "doc"],
+ Profile::User => &["dist", "build"],
+ };
+
+ println!();
+
+ t!(install_git_hook_maybe(&config));
+
+ println!();
+
+ println!("To get started, try one of the following commands:");
+ for cmd in suggestions {
+ println!("- `x.py {}`", cmd);
+ }
+
+ if profile != Profile::User {
+ println!(
+ "For more suggestions, see https://rustc-dev-guide.rust-lang.org/building/suggested.html"
+ );
+ }
+}
+
+fn rustup_installed() -> bool {
+ Command::new("rustup")
+ .arg("--version")
+ .stdout(std::process::Stdio::null())
+ .output()
+ .map_or(false, |output| output.status.success())
+}
+
+fn stage_dir_exists(stage_path: &str) -> bool {
+ match fs::create_dir(&stage_path) {
+ Ok(_) => true,
+ Err(_) => Path::new(&stage_path).exists(),
+ }
+}
+
+fn attempt_toolchain_link(stage_path: &str) {
+ if toolchain_is_linked() {
+ return;
+ }
+
+ if !ensure_stage1_toolchain_placeholder_exists(stage_path) {
+ eprintln!(
+ "Failed to create a template for stage 1 toolchain or confirm that it already exists"
+ );
+ return;
+ }
+
+ if try_link_toolchain(&stage_path) {
+ println!(
+ "Added `stage1` rustup toolchain; try `cargo +stage1 build` on a separate rust project to run a newly-built toolchain"
+ );
+ } else {
+ eprintln!("`rustup` failed to link stage 1 build to `stage1` toolchain");
+ eprintln!(
+ "To manually link stage 1 build to `stage1` toolchain, run:\n
+ `rustup toolchain link stage1 {}`",
+ &stage_path
+ );
+ }
+}
+
+fn toolchain_is_linked() -> bool {
+ match Command::new("rustup")
+ .args(&["toolchain", "list"])
+ .stdout(std::process::Stdio::piped())
+ .output()
+ {
+ Ok(toolchain_list) => {
+ if !String::from_utf8_lossy(&toolchain_list.stdout).contains("stage1") {
+ return false;
+ }
+ // The toolchain has already been linked.
+ println!(
+ "`stage1` toolchain already linked; not attempting to link `stage1` toolchain"
+ );
+ }
+ Err(_) => {
+ // In this case, we don't know if the `stage1` toolchain has been linked;
+ // but `rustup` failed, so let's not go any further.
+ println!(
+ "`rustup` failed to list current toolchains; not attempting to link `stage1` toolchain"
+ );
+ }
+ }
+ true
+}
+
+fn try_link_toolchain(stage_path: &str) -> bool {
+ Command::new("rustup")
+ .stdout(std::process::Stdio::null())
+ .args(&["toolchain", "link", "stage1", &stage_path])
+ .output()
+ .map_or(false, |output| output.status.success())
+}
+
+fn ensure_stage1_toolchain_placeholder_exists(stage_path: &str) -> bool {
+ let pathbuf = PathBuf::from(stage_path);
+
+ if fs::create_dir_all(pathbuf.join("lib")).is_err() {
+ return false;
+ };
+
+ let pathbuf = pathbuf.join("bin");
+ if fs::create_dir_all(&pathbuf).is_err() {
+ return false;
+ };
+
+ let pathbuf = pathbuf.join(format!("rustc{}", EXE_SUFFIX));
+
+ if pathbuf.exists() {
+ return true;
+ }
+
+ // Take care not to overwrite the file
+ let result = File::options().append(true).create(true).open(&pathbuf);
+ if result.is_err() {
+ return false;
+ }
+
+ return true;
+}
+
+// Used to get the path for `Subcommand::Setup`
+pub fn interactive_path() -> io::Result<Profile> {
+ fn abbrev_all() -> impl Iterator<Item = ((String, String), Profile)> {
+ ('a'..)
+ .zip(1..)
+ .map(|(letter, number)| (letter.to_string(), number.to_string()))
+ .zip(Profile::all())
+ }
+
+ fn parse_with_abbrev(input: &str) -> Result<Profile, String> {
+ let input = input.trim().to_lowercase();
+ for ((letter, number), profile) in abbrev_all() {
+ if input == letter || input == number {
+ return Ok(profile);
+ }
+ }
+ input.parse()
+ }
+
+ println!("Welcome to the Rust project! What do you want to do with x.py?");
+ for ((letter, _), profile) in abbrev_all() {
+ println!("{}) {}: {}", letter, profile, profile.purpose());
+ }
+ let template = loop {
+ print!(
+ "Please choose one ({}): ",
+ abbrev_all().map(|((l, _), _)| l).collect::<Vec<_>>().join("/")
+ );
+ io::stdout().flush()?;
+ let mut input = String::new();
+ io::stdin().read_line(&mut input)?;
+ if input.is_empty() {
+ eprintln!("EOF on stdin, when expecting answer to question. Giving up.");
+ crate::detail_exit(1);
+ }
+ break match parse_with_abbrev(&input) {
+ Ok(profile) => profile,
+ Err(err) => {
+ eprintln!("error: {}", err);
+ eprintln!("note: press Ctrl+C to exit");
+ continue;
+ }
+ };
+ };
+ Ok(template)
+}
+
+// install a git hook to automatically run tidy --bless, if they want
+fn install_git_hook_maybe(config: &Config) -> io::Result<()> {
+ let mut input = String::new();
+ println!(
+ "Rust's CI will automatically fail if it doesn't pass `tidy`, the internal tool for ensuring code quality.
+If you'd like, x.py can install a git hook for you that will automatically run `tidy --bless` before
+pushing your code to ensure your code is up to par. If you decide later that this behavior is
+undesirable, simply delete the `pre-push` file from .git/hooks."
+ );
+
+ let should_install = loop {
+ print!("Would you like to install the git hook?: [y/N] ");
+ io::stdout().flush()?;
+ input.clear();
+ io::stdin().read_line(&mut input)?;
+ break match input.trim().to_lowercase().as_str() {
+ "y" | "yes" => true,
+ "n" | "no" | "" => false,
+ _ => {
+ eprintln!("error: unrecognized option '{}'", input.trim());
+ eprintln!("note: press Ctrl+C to exit");
+ continue;
+ }
+ };
+ };
+
+ if should_install {
+ let src = config.src.join("src").join("etc").join("pre-push.sh");
+ let git =
+ t!(config.git().args(&["rev-parse", "--git-common-dir"]).output().map(|output| {
+ assert!(output.status.success(), "failed to run `git`");
+ PathBuf::from(t!(String::from_utf8(output.stdout)).trim())
+ }));
+ let dst = git.join("hooks").join("pre-push");
+ match fs::hard_link(src, &dst) {
+ Err(e) => eprintln!(
+ "error: could not create hook {}: do you already have the git hook installed?\n{}",
+ dst.display(),
+ e
+ ),
+ Ok(_) => println!("Linked `src/etc/pre-push.sh` to `.git/hooks/pre-push`"),
+ };
+ } else {
+ println!("Ok, skipping installation!");
+ }
+ Ok(())
+}
diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs
new file mode 100644
index 000000000..7b0c029c1
--- /dev/null
+++ b/src/bootstrap/tarball.rs
@@ -0,0 +1,374 @@
+use std::{
+ path::{Path, PathBuf},
+ process::Command,
+};
+
+use crate::builder::Builder;
+use crate::util::t;
+
+#[derive(Copy, Clone)]
+pub(crate) enum OverlayKind {
+ Rust,
+ LLVM,
+ Cargo,
+ Clippy,
+ Miri,
+ Rustfmt,
+ RustDemangler,
+ RLS,
+ RustAnalyzer,
+}
+
+impl OverlayKind {
+ fn legal_and_readme(&self) -> &[&str] {
+ match self {
+ OverlayKind::Rust => &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"],
+ OverlayKind::LLVM => {
+ &["src/llvm-project/llvm/LICENSE.TXT", "src/llvm-project/llvm/README.txt"]
+ }
+ OverlayKind::Cargo => &[
+ "src/tools/cargo/README.md",
+ "src/tools/cargo/LICENSE-MIT",
+ "src/tools/cargo/LICENSE-APACHE",
+ "src/tools/cargo/LICENSE-THIRD-PARTY",
+ ],
+ OverlayKind::Clippy => &[
+ "src/tools/clippy/README.md",
+ "src/tools/clippy/LICENSE-APACHE",
+ "src/tools/clippy/LICENSE-MIT",
+ ],
+ OverlayKind::Miri => &[
+ "src/tools/miri/README.md",
+ "src/tools/miri/LICENSE-APACHE",
+ "src/tools/miri/LICENSE-MIT",
+ ],
+ OverlayKind::Rustfmt => &[
+ "src/tools/rustfmt/README.md",
+ "src/tools/rustfmt/LICENSE-APACHE",
+ "src/tools/rustfmt/LICENSE-MIT",
+ ],
+ OverlayKind::RustDemangler => {
+ &["src/tools/rust-demangler/README.md", "LICENSE-APACHE", "LICENSE-MIT"]
+ }
+ OverlayKind::RLS => &[
+ "src/tools/rls/README.md",
+ "src/tools/rls/LICENSE-APACHE",
+ "src/tools/rls/LICENSE-MIT",
+ ],
+ OverlayKind::RustAnalyzer => &[
+ "src/tools/rust-analyzer/README.md",
+ "src/tools/rust-analyzer/LICENSE-APACHE",
+ "src/tools/rust-analyzer/LICENSE-MIT",
+ ],
+ }
+ }
+
+ fn version(&self, builder: &Builder<'_>) -> String {
+ match self {
+ OverlayKind::Rust => builder.rust_version(),
+ OverlayKind::LLVM => builder.rust_version(),
+ OverlayKind::RustDemangler => builder.release_num("rust-demangler"),
+ OverlayKind::Cargo => {
+ builder.cargo_info.version(builder, &builder.release_num("cargo"))
+ }
+ OverlayKind::Clippy => {
+ builder.clippy_info.version(builder, &builder.release_num("clippy"))
+ }
+ OverlayKind::Miri => builder.miri_info.version(builder, &builder.release_num("miri")),
+ OverlayKind::Rustfmt => {
+ builder.rustfmt_info.version(builder, &builder.release_num("rustfmt"))
+ }
+ OverlayKind::RLS => builder.rls_info.version(builder, &builder.release_num("rls")),
+ OverlayKind::RustAnalyzer => builder
+ .rust_analyzer_info
+ .version(builder, &builder.release_num("rust-analyzer/crates/rust-analyzer")),
+ }
+ }
+}
+
+pub(crate) struct Tarball<'a> {
+ builder: &'a Builder<'a>,
+
+ pkgname: String,
+ component: String,
+ target: Option<String>,
+ product_name: String,
+ overlay: OverlayKind,
+
+ temp_dir: PathBuf,
+ image_dir: PathBuf,
+ overlay_dir: PathBuf,
+ bulk_dirs: Vec<PathBuf>,
+
+ include_target_in_component_name: bool,
+ is_preview: bool,
+ permit_symlinks: bool,
+}
+
+impl<'a> Tarball<'a> {
+ pub(crate) fn new(builder: &'a Builder<'a>, component: &str, target: &str) -> Self {
+ Self::new_inner(builder, component, Some(target.into()))
+ }
+
+ pub(crate) fn new_targetless(builder: &'a Builder<'a>, component: &str) -> Self {
+ Self::new_inner(builder, component, None)
+ }
+
+ fn new_inner(builder: &'a Builder<'a>, component: &str, target: Option<String>) -> Self {
+ let pkgname = crate::dist::pkgname(builder, component);
+
+ let mut temp_dir = builder.out.join("tmp").join("tarball").join(component);
+ if let Some(target) = &target {
+ temp_dir = temp_dir.join(target);
+ }
+ let _ = std::fs::remove_dir_all(&temp_dir);
+
+ let image_dir = temp_dir.join("image");
+ let overlay_dir = temp_dir.join("overlay");
+
+ Self {
+ builder,
+
+ pkgname,
+ component: component.into(),
+ target,
+ product_name: "Rust".into(),
+ overlay: OverlayKind::Rust,
+
+ temp_dir,
+ image_dir,
+ overlay_dir,
+ bulk_dirs: Vec::new(),
+
+ include_target_in_component_name: false,
+ is_preview: false,
+ permit_symlinks: false,
+ }
+ }
+
+ pub(crate) fn set_overlay(&mut self, overlay: OverlayKind) {
+ self.overlay = overlay;
+ }
+
+ pub(crate) fn set_product_name(&mut self, name: &str) {
+ self.product_name = name.into();
+ }
+
+ pub(crate) fn include_target_in_component_name(&mut self, include: bool) {
+ self.include_target_in_component_name = include;
+ }
+
+ pub(crate) fn is_preview(&mut self, is: bool) {
+ self.is_preview = is;
+ }
+
+ pub(crate) fn permit_symlinks(&mut self, flag: bool) {
+ self.permit_symlinks = flag;
+ }
+
+ pub(crate) fn image_dir(&self) -> &Path {
+ t!(std::fs::create_dir_all(&self.image_dir));
+ &self.image_dir
+ }
+
+ pub(crate) fn add_file(&self, src: impl AsRef<Path>, destdir: impl AsRef<Path>, perms: u32) {
+ // create_dir_all fails to create `foo/bar/.`, so when the destination is "." this simply
+ // uses the base directory as the destination directory.
+ let destdir = if destdir.as_ref() == Path::new(".") {
+ self.image_dir.clone()
+ } else {
+ self.image_dir.join(destdir.as_ref())
+ };
+
+ t!(std::fs::create_dir_all(&destdir));
+ self.builder.install(src.as_ref(), &destdir, perms);
+ }
+
+ pub(crate) fn add_renamed_file(
+ &self,
+ src: impl AsRef<Path>,
+ destdir: impl AsRef<Path>,
+ new_name: &str,
+ ) {
+ let destdir = self.image_dir.join(destdir.as_ref());
+ t!(std::fs::create_dir_all(&destdir));
+ self.builder.copy(src.as_ref(), &destdir.join(new_name));
+ }
+
+ pub(crate) fn add_legal_and_readme_to(&self, destdir: impl AsRef<Path>) {
+ for file in self.overlay.legal_and_readme() {
+ self.add_file(self.builder.src.join(file), destdir.as_ref(), 0o644);
+ }
+ }
+
+ pub(crate) fn add_dir(&self, src: impl AsRef<Path>, dest: impl AsRef<Path>) {
+ let dest = self.image_dir.join(dest.as_ref());
+
+ t!(std::fs::create_dir_all(&dest));
+ self.builder.cp_r(src.as_ref(), &dest);
+ }
+
+ pub(crate) fn add_bulk_dir(&mut self, src: impl AsRef<Path>, dest: impl AsRef<Path>) {
+ self.bulk_dirs.push(dest.as_ref().to_path_buf());
+ self.add_dir(src, dest);
+ }
+
+ pub(crate) fn generate(self) -> GeneratedTarball {
+ let mut component_name = self.component.clone();
+ if self.is_preview {
+ component_name.push_str("-preview");
+ }
+ if self.include_target_in_component_name {
+ component_name.push('-');
+ component_name.push_str(
+ &self
+ .target
+ .as_ref()
+ .expect("include_target_in_component_name used in a targetless tarball"),
+ );
+ }
+
+ self.run(|this, cmd| {
+ cmd.arg("generate")
+ .arg("--image-dir")
+ .arg(&this.image_dir)
+ .arg(format!("--component-name={}", &component_name));
+
+ if let Some((dir, dirs)) = this.bulk_dirs.split_first() {
+ let mut arg = dir.as_os_str().to_os_string();
+ for dir in dirs {
+ arg.push(",");
+ arg.push(dir);
+ }
+ cmd.arg("--bulk-dirs").arg(&arg);
+ }
+
+ this.non_bare_args(cmd);
+ })
+ }
+
+ pub(crate) fn combine(self, tarballs: &[GeneratedTarball]) -> GeneratedTarball {
+ let mut input_tarballs = tarballs[0].path.as_os_str().to_os_string();
+ for tarball in &tarballs[1..] {
+ input_tarballs.push(",");
+ input_tarballs.push(&tarball.path);
+ }
+
+ self.run(|this, cmd| {
+ cmd.arg("combine").arg("--input-tarballs").arg(input_tarballs);
+ this.non_bare_args(cmd);
+ })
+ }
+
+ pub(crate) fn bare(self) -> GeneratedTarball {
+ // Bare tarballs should have the top level directory match the package
+ // name, not "image". We rename the image directory just before passing
+ // into rust-installer.
+ let dest = self.temp_dir.join(self.package_name());
+ t!(std::fs::rename(&self.image_dir, &dest));
+
+ self.run(|this, cmd| {
+ let distdir = crate::dist::distdir(this.builder);
+ t!(std::fs::create_dir_all(&distdir));
+ cmd.arg("tarball")
+ .arg("--input")
+ .arg(&dest)
+ .arg("--output")
+ .arg(distdir.join(this.package_name()));
+ })
+ }
+
+ fn package_name(&self) -> String {
+ if let Some(target) = &self.target {
+ format!("{}-{}", self.pkgname, target)
+ } else {
+ self.pkgname.clone()
+ }
+ }
+
+ fn non_bare_args(&self, cmd: &mut Command) {
+ cmd.arg("--rel-manifest-dir=rustlib")
+ .arg("--legacy-manifest-dirs=rustlib,cargo")
+ .arg(format!("--product-name={}", self.product_name))
+ .arg(format!("--success-message={} installed.", self.component))
+ .arg(format!("--package-name={}", self.package_name()))
+ .arg("--non-installed-overlay")
+ .arg(&self.overlay_dir)
+ .arg("--output-dir")
+ .arg(crate::dist::distdir(self.builder));
+ }
+
+ fn run(self, build_cli: impl FnOnce(&Tarball<'a>, &mut Command)) -> GeneratedTarball {
+ t!(std::fs::create_dir_all(&self.overlay_dir));
+ self.builder.create(&self.overlay_dir.join("version"), &self.overlay.version(self.builder));
+ if let Some(sha) = self.builder.rust_sha() {
+ self.builder.create(&self.overlay_dir.join("git-commit-hash"), &sha);
+ }
+ for file in self.overlay.legal_and_readme() {
+ self.builder.install(&self.builder.src.join(file), &self.overlay_dir, 0o644);
+ }
+
+ let mut cmd = self.builder.tool_cmd(crate::tool::Tool::RustInstaller);
+
+ let package_name = self.package_name();
+ self.builder.info(&format!("Dist {}", package_name));
+ let _time = crate::util::timeit(self.builder);
+
+ build_cli(&self, &mut cmd);
+ cmd.arg("--work-dir").arg(&self.temp_dir);
+ if let Some(formats) = &self.builder.config.dist_compression_formats {
+ assert!(!formats.is_empty(), "dist.compression-formats can't be empty");
+ cmd.arg("--compression-formats").arg(formats.join(","));
+ }
+ self.builder.run(&mut cmd);
+
+ // Ensure there are no symbolic links in the tarball. In particular,
+ // rustup-toolchain-install-master and most versions of Windows can't handle symbolic links.
+ let decompressed_output = self.temp_dir.join(&package_name);
+ if !self.builder.config.dry_run && !self.permit_symlinks {
+ for entry in walkdir::WalkDir::new(&decompressed_output) {
+ let entry = t!(entry);
+ if entry.path_is_symlink() {
+ panic!("generated a symlink in a tarball: {}", entry.path().display());
+ }
+ }
+ }
+
+ // Use either the first compression format defined, or "gz" as the default.
+ let ext = self
+ .builder
+ .config
+ .dist_compression_formats
+ .as_ref()
+ .and_then(|formats| formats.get(0))
+ .map(|s| s.as_str())
+ .unwrap_or("gz");
+
+ GeneratedTarball {
+ path: crate::dist::distdir(self.builder).join(format!("{}.tar.{}", package_name, ext)),
+ decompressed_output,
+ work: self.temp_dir,
+ }
+ }
+}
+
+#[derive(Debug, Clone)]
+pub struct GeneratedTarball {
+ path: PathBuf,
+ decompressed_output: PathBuf,
+ work: PathBuf,
+}
+
+impl GeneratedTarball {
+ pub(crate) fn tarball(&self) -> &Path {
+ &self.path
+ }
+
+ pub(crate) fn decompressed_output(&self) -> &Path {
+ &self.decompressed_output
+ }
+
+ pub(crate) fn work_dir(&self) -> &Path {
+ &self.work
+ }
+}
diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs
new file mode 100644
index 000000000..c0fa8c9ac
--- /dev/null
+++ b/src/bootstrap/test.rs
@@ -0,0 +1,2542 @@
+//! Implementation of the test-related targets of the build system.
+//!
+//! This file implements the various regression test suites that we execute on
+//! our CI.
+
+use std::env;
+use std::ffi::OsString;
+use std::fmt;
+use std::fs;
+use std::iter;
+use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
+
+use crate::builder::{Builder, Compiler, Kind, RunConfig, ShouldRun, Step};
+use crate::cache::Interned;
+use crate::compile;
+use crate::config::TargetSelection;
+use crate::dist;
+use crate::flags::Subcommand;
+use crate::native;
+use crate::tool::{self, SourceType, Tool};
+use crate::toolstate::ToolState;
+use crate::util::{self, add_link_lib_path, dylib_path, dylib_path_var, output, t};
+use crate::{envify, CLang, DocTests, GitRepo, Mode};
+
+const ADB_TEST_DIR: &str = "/data/tmp/work";
+
+/// The two modes of the test runner; tests or benchmarks.
+#[derive(Debug, PartialEq, Eq, Hash, Copy, Clone, PartialOrd, Ord)]
+pub enum TestKind {
+ /// Run `cargo test`.
+ Test,
+ /// Run `cargo bench`.
+ Bench,
+}
+
+impl From<Kind> for TestKind {
+ fn from(kind: Kind) -> Self {
+ match kind {
+ Kind::Test => TestKind::Test,
+ Kind::Bench => TestKind::Bench,
+ _ => panic!("unexpected kind in crate: {:?}", kind),
+ }
+ }
+}
+
+impl TestKind {
+ // Return the cargo subcommand for this test kind
+ fn subcommand(self) -> &'static str {
+ match self {
+ TestKind::Test => "test",
+ TestKind::Bench => "bench",
+ }
+ }
+}
+
+impl fmt::Display for TestKind {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.write_str(match *self {
+ TestKind::Test => "Testing",
+ TestKind::Bench => "Benchmarking",
+ })
+ }
+}
+
+fn try_run(builder: &Builder<'_>, cmd: &mut Command) -> bool {
+ if !builder.fail_fast {
+ if !builder.try_run(cmd) {
+ let mut failures = builder.delayed_failures.borrow_mut();
+ failures.push(format!("{:?}", cmd));
+ return false;
+ }
+ } else {
+ builder.run(cmd);
+ }
+ true
+}
+
+fn try_run_quiet(builder: &Builder<'_>, cmd: &mut Command) -> bool {
+ if !builder.fail_fast {
+ if !builder.try_run_quiet(cmd) {
+ let mut failures = builder.delayed_failures.borrow_mut();
+ failures.push(format!("{:?}", cmd));
+ return false;
+ }
+ } else {
+ builder.run_quiet(cmd);
+ }
+ true
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct Linkcheck {
+ host: TargetSelection,
+}
+
+impl Step for Linkcheck {
+ type Output = ();
+ const ONLY_HOSTS: bool = true;
+ const DEFAULT: bool = true;
+
+ /// Runs the `linkchecker` tool as compiled in `stage` by the `host` compiler.
+ ///
+ /// This tool in `src/tools` will verify the validity of all our links in the
+ /// documentation to ensure we don't have a bunch of dead ones.
+ fn run(self, builder: &Builder<'_>) {
+ let host = self.host;
+ let hosts = &builder.hosts;
+ let targets = &builder.targets;
+
+ // if we have different hosts and targets, some things may be built for
+ // the host (e.g. rustc) and others for the target (e.g. std). The
+ // documentation built for each will contain broken links to
+ // docs built for the other platform (e.g. rustc linking to cargo)
+ if (hosts != targets) && !hosts.is_empty() && !targets.is_empty() {
+ panic!(
+ "Linkcheck currently does not support builds with different hosts and targets.
+You can skip linkcheck with --exclude src/tools/linkchecker"
+ );
+ }
+
+ builder.info(&format!("Linkcheck ({})", host));
+
+ // Test the linkchecker itself.
+ let bootstrap_host = builder.config.build;
+ let compiler = builder.compiler(0, bootstrap_host);
+ let cargo = tool::prepare_tool_cargo(
+ builder,
+ compiler,
+ Mode::ToolBootstrap,
+ bootstrap_host,
+ "test",
+ "src/tools/linkchecker",
+ SourceType::InTree,
+ &[],
+ );
+ try_run(builder, &mut cargo.into());
+
+ // Build all the default documentation.
+ builder.default_doc(&[]);
+
+ // Run the linkchecker.
+ let _time = util::timeit(&builder);
+ try_run(
+ builder,
+ builder.tool_cmd(Tool::Linkchecker).arg(builder.out.join(host.triple).join("doc")),
+ );
+ }
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let builder = run.builder;
+ let run = run.path("src/tools/linkchecker");
+ run.default_condition(builder.config.docs)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Linkcheck { host: run.target });
+ }
+}
+
+fn check_if_tidy_is_installed() -> bool {
+ Command::new("tidy")
+ .arg("--version")
+ .stdout(Stdio::null())
+ .status()
+ .map_or(false, |status| status.success())
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct HtmlCheck {
+ target: TargetSelection,
+}
+
+impl Step for HtmlCheck {
+ type Output = ();
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let run = run.path("src/tools/html-checker");
+ run.lazy_default_condition(Box::new(check_if_tidy_is_installed))
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(HtmlCheck { target: run.target });
+ }
+
+ fn run(self, builder: &Builder<'_>) {
+ if !check_if_tidy_is_installed() {
+ eprintln!("not running HTML-check tool because `tidy` is missing");
+ eprintln!(
+ "Note that `tidy` is not the in-tree `src/tools/tidy` but needs to be installed"
+ );
+ panic!("Cannot run html-check tests");
+ }
+ // Ensure that a few different kinds of documentation are available.
+ builder.default_doc(&[]);
+ builder.ensure(crate::doc::Rustc { target: self.target, stage: builder.top_stage });
+
+ try_run(builder, builder.tool_cmd(Tool::HtmlChecker).arg(builder.doc_out(self.target)));
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct Cargotest {
+ stage: u32,
+ host: TargetSelection,
+}
+
+impl Step for Cargotest {
+ type Output = ();
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/tools/cargotest")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Cargotest { stage: run.builder.top_stage, host: run.target });
+ }
+
+ /// Runs the `cargotest` tool as compiled in `stage` by the `host` compiler.
+ ///
+ /// This tool in `src/tools` will check out a few Rust projects and run `cargo
+ /// test` to ensure that we don't regress the test suites there.
+ fn run(self, builder: &Builder<'_>) {
+ let compiler = builder.compiler(self.stage, self.host);
+ builder.ensure(compile::Rustc::new(compiler, compiler.host));
+ let cargo = builder.ensure(tool::Cargo { compiler, target: compiler.host });
+
+ // Note that this is a short, cryptic, and not scoped directory name. This
+ // is currently to minimize the length of path on Windows where we otherwise
+ // quickly run into path name limit constraints.
+ let out_dir = builder.out.join("ct");
+ t!(fs::create_dir_all(&out_dir));
+
+ let _time = util::timeit(&builder);
+ let mut cmd = builder.tool_cmd(Tool::CargoTest);
+ try_run(
+ builder,
+ cmd.arg(&cargo)
+ .arg(&out_dir)
+ .args(builder.config.cmd.test_args())
+ .env("RUSTC", builder.rustc(compiler))
+ .env("RUSTDOC", builder.rustdoc(compiler)),
+ );
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct Cargo {
+ stage: u32,
+ host: TargetSelection,
+}
+
+impl Step for Cargo {
+ type Output = ();
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/tools/cargo")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Cargo { stage: run.builder.top_stage, host: run.target });
+ }
+
+ /// Runs `cargo test` for `cargo` packaged with Rust.
+ fn run(self, builder: &Builder<'_>) {
+ let compiler = builder.compiler(self.stage, self.host);
+
+ builder.ensure(tool::Cargo { compiler, target: self.host });
+ let mut cargo = tool::prepare_tool_cargo(
+ builder,
+ compiler,
+ Mode::ToolRustc,
+ self.host,
+ "test",
+ "src/tools/cargo",
+ SourceType::Submodule,
+ &[],
+ );
+
+ if !builder.fail_fast {
+ cargo.arg("--no-fail-fast");
+ }
+ cargo.arg("--").args(builder.config.cmd.test_args());
+
+ // Don't run cross-compile tests, we may not have cross-compiled libstd libs
+ // available.
+ cargo.env("CFG_DISABLE_CROSS_TESTS", "1");
+ // Forcibly disable tests using nightly features since any changes to
+ // those features won't be able to land.
+ cargo.env("CARGO_TEST_DISABLE_NIGHTLY", "1");
+
+ cargo.env("PATH", &path_for_cargo(builder, compiler));
+
+ try_run(builder, &mut cargo.into());
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct Rls {
+ stage: u32,
+ host: TargetSelection,
+}
+
+impl Step for Rls {
+ type Output = ();
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/tools/rls")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Rls { stage: run.builder.top_stage, host: run.target });
+ }
+
+ /// Runs `cargo test` for the rls.
+ fn run(self, builder: &Builder<'_>) {
+ let stage = self.stage;
+ let host = self.host;
+ let compiler = builder.compiler(stage, host);
+
+ let build_result =
+ builder.ensure(tool::Rls { compiler, target: self.host, extra_features: Vec::new() });
+ if build_result.is_none() {
+ eprintln!("failed to test rls: could not build");
+ return;
+ }
+
+ let mut cargo = tool::prepare_tool_cargo(
+ builder,
+ compiler,
+ Mode::ToolRustc,
+ host,
+ "test",
+ "src/tools/rls",
+ SourceType::Submodule,
+ &[],
+ );
+
+ cargo.add_rustc_lib_path(builder, compiler);
+ cargo.arg("--").args(builder.config.cmd.test_args());
+
+ if try_run(builder, &mut cargo.into()) {
+ builder.save_toolstate("rls", ToolState::TestPass);
+ }
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct RustAnalyzer {
+ stage: u32,
+ host: TargetSelection,
+}
+
+impl Step for RustAnalyzer {
+ type Output = ();
+ const ONLY_HOSTS: bool = true;
+ const DEFAULT: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/tools/rust-analyzer")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Self { stage: run.builder.top_stage, host: run.target });
+ }
+
+ /// Runs `cargo test` for rust-analyzer
+ fn run(self, builder: &Builder<'_>) {
+ let stage = self.stage;
+ let host = self.host;
+ let compiler = builder.compiler(stage, host);
+
+ builder.ensure(tool::RustAnalyzer { compiler, target: self.host }).expect("in-tree tool");
+
+ let workspace_path = "src/tools/rust-analyzer";
+ // until the whole RA test suite runs on `i686`, we only run
+ // `proc-macro-srv` tests
+ let crate_path = "src/tools/rust-analyzer/crates/proc-macro-srv";
+ let mut cargo = tool::prepare_tool_cargo(
+ builder,
+ compiler,
+ Mode::ToolStd,
+ host,
+ "test",
+ crate_path,
+ SourceType::InTree,
+ &["sysroot-abi".to_owned()],
+ );
+
+ let dir = builder.src.join(workspace_path);
+ // needed by rust-analyzer to find its own text fixtures, cf.
+ // https://github.com/rust-analyzer/expect-test/issues/33
+ cargo.env("CARGO_WORKSPACE_DIR", &dir);
+
+ // RA's test suite tries to write to the source directory, that can't
+ // work in Rust CI
+ cargo.env("SKIP_SLOW_TESTS", "1");
+
+ cargo.add_rustc_lib_path(builder, compiler);
+ cargo.arg("--").args(builder.config.cmd.test_args());
+
+ builder.run(&mut cargo.into());
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct Rustfmt {
+ stage: u32,
+ host: TargetSelection,
+}
+
+impl Step for Rustfmt {
+ type Output = ();
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/tools/rustfmt")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Rustfmt { stage: run.builder.top_stage, host: run.target });
+ }
+
+ /// Runs `cargo test` for rustfmt.
+ fn run(self, builder: &Builder<'_>) {
+ let stage = self.stage;
+ let host = self.host;
+ let compiler = builder.compiler(stage, host);
+
+ builder
+ .ensure(tool::Rustfmt { compiler, target: self.host, extra_features: Vec::new() })
+ .expect("in-tree tool");
+
+ let mut cargo = tool::prepare_tool_cargo(
+ builder,
+ compiler,
+ Mode::ToolRustc,
+ host,
+ "test",
+ "src/tools/rustfmt",
+ SourceType::InTree,
+ &[],
+ );
+
+ let dir = testdir(builder, compiler.host);
+ t!(fs::create_dir_all(&dir));
+ cargo.env("RUSTFMT_TEST_DIR", dir);
+
+ cargo.add_rustc_lib_path(builder, compiler);
+
+ builder.run(&mut cargo.into());
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct RustDemangler {
+ stage: u32,
+ host: TargetSelection,
+}
+
+impl Step for RustDemangler {
+ type Output = ();
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/tools/rust-demangler")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(RustDemangler { stage: run.builder.top_stage, host: run.target });
+ }
+
+ /// Runs `cargo test` for rust-demangler.
+ fn run(self, builder: &Builder<'_>) {
+ let stage = self.stage;
+ let host = self.host;
+ let compiler = builder.compiler(stage, host);
+
+ let rust_demangler = builder
+ .ensure(tool::RustDemangler { compiler, target: self.host, extra_features: Vec::new() })
+ .expect("in-tree tool");
+ let mut cargo = tool::prepare_tool_cargo(
+ builder,
+ compiler,
+ Mode::ToolRustc,
+ host,
+ "test",
+ "src/tools/rust-demangler",
+ SourceType::InTree,
+ &[],
+ );
+
+ let dir = testdir(builder, compiler.host);
+ t!(fs::create_dir_all(&dir));
+
+ cargo.env("RUST_DEMANGLER_DRIVER_PATH", rust_demangler);
+
+ cargo.arg("--").args(builder.config.cmd.test_args());
+
+ cargo.add_rustc_lib_path(builder, compiler);
+
+ builder.run(&mut cargo.into());
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct Miri {
+ stage: u32,
+ host: TargetSelection,
+}
+
+impl Step for Miri {
+ type Output = ();
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/tools/miri")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Miri { stage: run.builder.top_stage, host: run.target });
+ }
+
+ /// Runs `cargo test` for miri.
+ fn run(self, builder: &Builder<'_>) {
+ let stage = self.stage;
+ let host = self.host;
+ let compiler = builder.compiler(stage, host);
+ // We need the stdlib for the *next* stage, as it was built with this compiler that also built Miri.
+ // Except if we are at stage 2, the bootstrap loop is complete and we can stick with our current stage.
+ let compiler_std = builder.compiler(if stage < 2 { stage + 1 } else { stage }, host);
+
+ let miri =
+ builder.ensure(tool::Miri { compiler, target: self.host, extra_features: Vec::new() });
+ let cargo_miri = builder.ensure(tool::CargoMiri {
+ compiler,
+ target: self.host,
+ extra_features: Vec::new(),
+ });
+ // The stdlib we need might be at a different stage. And just asking for the
+ // sysroot does not seem to populate it, so we do that first.
+ builder.ensure(compile::Std::new(compiler_std, host));
+ let sysroot = builder.sysroot(compiler_std);
+ if let (Some(miri), Some(_cargo_miri)) = (miri, cargo_miri) {
+ let mut cargo =
+ builder.cargo(compiler, Mode::ToolRustc, SourceType::Submodule, host, "install");
+ cargo.arg("xargo");
+ // Configure `cargo install` path. cargo adds a `bin/`.
+ cargo.env("CARGO_INSTALL_ROOT", &builder.out);
+
+ let mut cargo = Command::from(cargo);
+ if !try_run(builder, &mut cargo) {
+ return;
+ }
+
+ // # Run `cargo miri setup`.
+ let mut cargo = tool::prepare_tool_cargo(
+ builder,
+ compiler,
+ Mode::ToolRustc,
+ host,
+ "run",
+ "src/tools/miri/cargo-miri",
+ SourceType::Submodule,
+ &[],
+ );
+ cargo.add_rustc_lib_path(builder, compiler);
+ cargo.arg("--").arg("miri").arg("setup");
+
+ // Tell `cargo miri setup` where to find the sources.
+ cargo.env("XARGO_RUST_SRC", builder.src.join("library"));
+ // Tell it where to find Miri.
+ cargo.env("MIRI", &miri);
+ // Debug things.
+ cargo.env("RUST_BACKTRACE", "1");
+ // Let cargo-miri know where xargo ended up.
+ cargo.env("XARGO_CHECK", builder.out.join("bin").join("xargo-check"));
+
+ let mut cargo = Command::from(cargo);
+ if !try_run(builder, &mut cargo) {
+ return;
+ }
+
+ // # Determine where Miri put its sysroot.
+ // To this end, we run `cargo miri setup --print-sysroot` and capture the output.
+ // (We do this separately from the above so that when the setup actually
+ // happens we get some output.)
+ // We re-use the `cargo` from above.
+ cargo.arg("--print-sysroot");
+
+ // FIXME: Is there a way in which we can re-use the usual `run` helpers?
+ let miri_sysroot = if builder.config.dry_run {
+ String::new()
+ } else {
+ builder.verbose(&format!("running: {:?}", cargo));
+ let out = cargo
+ .output()
+ .expect("We already ran `cargo miri setup` before and that worked");
+ assert!(out.status.success(), "`cargo miri setup` returned with non-0 exit code");
+ // Output is "<sysroot>\n".
+ let stdout = String::from_utf8(out.stdout)
+ .expect("`cargo miri setup` stdout is not valid UTF-8");
+ let sysroot = stdout.trim_end();
+ builder.verbose(&format!("`cargo miri setup --print-sysroot` said: {:?}", sysroot));
+ sysroot.to_owned()
+ };
+
+ // # Run `cargo test`.
+ let mut cargo = tool::prepare_tool_cargo(
+ builder,
+ compiler,
+ Mode::ToolRustc,
+ host,
+ "test",
+ "src/tools/miri",
+ SourceType::Submodule,
+ &[],
+ );
+ cargo.add_rustc_lib_path(builder, compiler);
+
+ // miri tests need to know about the stage sysroot
+ cargo.env("MIRI_SYSROOT", miri_sysroot);
+ cargo.env("MIRI_HOST_SYSROOT", sysroot);
+ cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler));
+ cargo.env("MIRI", miri);
+
+ cargo.arg("--").args(builder.config.cmd.test_args());
+
+ let mut cargo = Command::from(cargo);
+ if !try_run(builder, &mut cargo) {
+ return;
+ }
+
+ // # Done!
+ builder.save_toolstate("miri", ToolState::TestPass);
+ } else {
+ eprintln!("failed to test miri: could not build");
+ }
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct CompiletestTest {
+ host: TargetSelection,
+}
+
+impl Step for CompiletestTest {
+ type Output = ();
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/tools/compiletest")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(CompiletestTest { host: run.target });
+ }
+
+ /// Runs `cargo test` for compiletest.
+ fn run(self, builder: &Builder<'_>) {
+ let host = self.host;
+ let compiler = builder.compiler(0, host);
+
+ // We need `ToolStd` for the locally-built sysroot because
+ // compiletest uses unstable features of the `test` crate.
+ builder.ensure(compile::Std::new(compiler, host));
+ let cargo = tool::prepare_tool_cargo(
+ builder,
+ compiler,
+ Mode::ToolStd,
+ host,
+ "test",
+ "src/tools/compiletest",
+ SourceType::InTree,
+ &[],
+ );
+
+ try_run(builder, &mut cargo.into());
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct Clippy {
+ stage: u32,
+ host: TargetSelection,
+}
+
+impl Step for Clippy {
+ type Output = ();
+ const ONLY_HOSTS: bool = true;
+ const DEFAULT: bool = false;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/tools/clippy")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Clippy { stage: run.builder.top_stage, host: run.target });
+ }
+
+ /// Runs `cargo test` for clippy.
+ fn run(self, builder: &Builder<'_>) {
+ let stage = self.stage;
+ let host = self.host;
+ let compiler = builder.compiler(stage, host);
+
+ builder
+ .ensure(tool::Clippy { compiler, target: self.host, extra_features: Vec::new() })
+ .expect("in-tree tool");
+ let mut cargo = tool::prepare_tool_cargo(
+ builder,
+ compiler,
+ Mode::ToolRustc,
+ host,
+ "test",
+ "src/tools/clippy",
+ SourceType::InTree,
+ &[],
+ );
+
+ cargo.env("RUSTC_TEST_SUITE", builder.rustc(compiler));
+ cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler));
+ let host_libs = builder.stage_out(compiler, Mode::ToolRustc).join(builder.cargo_dir());
+ cargo.env("HOST_LIBS", host_libs);
+
+ cargo.arg("--").args(builder.config.cmd.test_args());
+
+ cargo.add_rustc_lib_path(builder, compiler);
+
+ if builder.try_run(&mut cargo.into()) {
+ // The tests succeeded; nothing to do.
+ return;
+ }
+
+ if !builder.config.cmd.bless() {
+ crate::detail_exit(1);
+ }
+
+ let mut cargo = builder.cargo(compiler, Mode::ToolRustc, SourceType::InTree, host, "run");
+ cargo.arg("-p").arg("clippy_dev");
+ // clippy_dev gets confused if it can't find `clippy/Cargo.toml`
+ cargo.current_dir(&builder.src.join("src").join("tools").join("clippy"));
+ if builder.config.rust_optimize {
+ cargo.env("PROFILE", "release");
+ } else {
+ cargo.env("PROFILE", "debug");
+ }
+ cargo.arg("--");
+ cargo.arg("bless");
+ builder.run(&mut cargo.into());
+ }
+}
+
+fn path_for_cargo(builder: &Builder<'_>, compiler: Compiler) -> OsString {
+ // Configure PATH to find the right rustc. NB. we have to use PATH
+ // and not RUSTC because the Cargo test suite has tests that will
+ // fail if rustc is not spelled `rustc`.
+ let path = builder.sysroot(compiler).join("bin");
+ let old_path = env::var_os("PATH").unwrap_or_default();
+ env::join_paths(iter::once(path).chain(env::split_paths(&old_path))).expect("")
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RustdocTheme {
+ pub compiler: Compiler,
+}
+
+impl Step for RustdocTheme {
+ type Output = ();
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/tools/rustdoc-themes")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ let compiler = run.builder.compiler(run.builder.top_stage, run.target);
+
+ run.builder.ensure(RustdocTheme { compiler });
+ }
+
+ fn run(self, builder: &Builder<'_>) {
+ let rustdoc = builder.bootstrap_out.join("rustdoc");
+ let mut cmd = builder.tool_cmd(Tool::RustdocTheme);
+ cmd.arg(rustdoc.to_str().unwrap())
+ .arg(builder.src.join("src/librustdoc/html/static/css/themes").to_str().unwrap())
+ .env("RUSTC_STAGE", self.compiler.stage.to_string())
+ .env("RUSTC_SYSROOT", builder.sysroot(self.compiler))
+ .env("RUSTDOC_LIBDIR", builder.sysroot_libdir(self.compiler, self.compiler.host))
+ .env("CFG_RELEASE_CHANNEL", &builder.config.channel)
+ .env("RUSTDOC_REAL", builder.rustdoc(self.compiler))
+ .env("RUSTC_BOOTSTRAP", "1");
+ if let Some(linker) = builder.linker(self.compiler.host) {
+ cmd.env("RUSTDOC_LINKER", linker);
+ }
+ if builder.is_fuse_ld_lld(self.compiler.host) {
+ cmd.env("RUSTDOC_FUSE_LD_LLD", "1");
+ }
+ try_run(builder, &mut cmd);
+ }
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RustdocJSStd {
+ pub target: TargetSelection,
+}
+
+impl Step for RustdocJSStd {
+ type Output = ();
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.suite_path("src/test/rustdoc-js-std")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(RustdocJSStd { target: run.target });
+ }
+
+ fn run(self, builder: &Builder<'_>) {
+ if let Some(ref nodejs) = builder.config.nodejs {
+ let mut command = Command::new(nodejs);
+ command
+ .arg(builder.src.join("src/tools/rustdoc-js/tester.js"))
+ .arg("--crate-name")
+ .arg("std")
+ .arg("--resource-suffix")
+ .arg(&builder.version)
+ .arg("--doc-folder")
+ .arg(builder.doc_out(self.target))
+ .arg("--test-folder")
+ .arg(builder.src.join("src/test/rustdoc-js-std"));
+ for path in &builder.paths {
+ if let Some(p) =
+ util::is_valid_test_suite_arg(path, "src/test/rustdoc-js-std", builder)
+ {
+ if !p.ends_with(".js") {
+ eprintln!("A non-js file was given: `{}`", path.display());
+ panic!("Cannot run rustdoc-js-std tests");
+ }
+ command.arg("--test-file").arg(path);
+ }
+ }
+ builder.ensure(crate::doc::Std { target: self.target, stage: builder.top_stage });
+ builder.run(&mut command);
+ } else {
+ builder.info("No nodejs found, skipping \"src/test/rustdoc-js-std\" tests");
+ }
+ }
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RustdocJSNotStd {
+ pub target: TargetSelection,
+ pub compiler: Compiler,
+}
+
+impl Step for RustdocJSNotStd {
+ type Output = ();
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.suite_path("src/test/rustdoc-js")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple());
+ run.builder.ensure(RustdocJSNotStd { target: run.target, compiler });
+ }
+
+ fn run(self, builder: &Builder<'_>) {
+ if builder.config.nodejs.is_some() {
+ builder.ensure(Compiletest {
+ compiler: self.compiler,
+ target: self.target,
+ mode: "js-doc-test",
+ suite: "rustdoc-js",
+ path: "src/test/rustdoc-js",
+ compare_mode: None,
+ });
+ } else {
+ builder.info("No nodejs found, skipping \"src/test/rustdoc-js\" tests");
+ }
+ }
+}
+
+fn get_browser_ui_test_version_inner(npm: &Path, global: bool) -> Option<String> {
+ let mut command = Command::new(&npm);
+ command.arg("list").arg("--parseable").arg("--long").arg("--depth=0");
+ if global {
+ command.arg("--global");
+ }
+ let lines = command
+ .output()
+ .map(|output| String::from_utf8_lossy(&output.stdout).into_owned())
+ .unwrap_or(String::new());
+ lines.lines().find_map(|l| l.split(":browser-ui-test@").skip(1).next()).map(|v| v.to_owned())
+}
+
+fn get_browser_ui_test_version(npm: &Path) -> Option<String> {
+ get_browser_ui_test_version_inner(npm, false)
+ .or_else(|| get_browser_ui_test_version_inner(npm, true))
+}
+
+fn compare_browser_ui_test_version(installed_version: &str, src: &Path) {
+ match fs::read_to_string(
+ src.join("src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version"),
+ ) {
+ Ok(v) => {
+ if v.trim() != installed_version {
+ eprintln!(
+ "⚠️ Installed version of browser-ui-test (`{}`) is different than the \
+ one used in the CI (`{}`)",
+ installed_version, v
+ );
+ }
+ }
+ Err(e) => eprintln!("Couldn't find the CI browser-ui-test version: {:?}", e),
+ }
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RustdocGUI {
+ pub target: TargetSelection,
+ pub compiler: Compiler,
+}
+
+impl Step for RustdocGUI {
+ type Output = ();
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let builder = run.builder;
+ let run = run.suite_path("src/test/rustdoc-gui");
+ run.lazy_default_condition(Box::new(move || {
+ builder.config.nodejs.is_some()
+ && builder
+ .config
+ .npm
+ .as_ref()
+ .map(|p| get_browser_ui_test_version(p).is_some())
+ .unwrap_or(false)
+ }))
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple());
+ run.builder.ensure(RustdocGUI { target: run.target, compiler });
+ }
+
+ fn run(self, builder: &Builder<'_>) {
+ let nodejs = builder.config.nodejs.as_ref().expect("nodejs isn't available");
+ let npm = builder.config.npm.as_ref().expect("npm isn't available");
+
+ builder.ensure(compile::Std::new(self.compiler, self.target));
+
+ // The goal here is to check if the necessary packages are installed, and if not, we
+ // panic.
+ match get_browser_ui_test_version(&npm) {
+ Some(version) => {
+ // We also check the version currently used in CI and emit a warning if it's not the
+ // same one.
+ compare_browser_ui_test_version(&version, &builder.build.src);
+ }
+ None => {
+ eprintln!(
+ "error: rustdoc-gui test suite cannot be run because npm `browser-ui-test` \
+ dependency is missing",
+ );
+ eprintln!(
+ "If you want to install the `{0}` dependency, run `npm install {0}`",
+ "browser-ui-test",
+ );
+ panic!("Cannot run rustdoc-gui tests");
+ }
+ }
+
+ let out_dir = builder.test_out(self.target).join("rustdoc-gui");
+
+ // We remove existing folder to be sure there won't be artifacts remaining.
+ builder.clear_if_dirty(&out_dir, &builder.rustdoc(self.compiler));
+
+ let src_path = builder.build.src.join("src/test/rustdoc-gui/src");
+ // We generate docs for the libraries present in the rustdoc-gui's src folder.
+ for entry in src_path.read_dir().expect("read_dir call failed") {
+ if let Ok(entry) = entry {
+ let path = entry.path();
+
+ if !path.is_dir() {
+ continue;
+ }
+
+ let mut cargo = Command::new(&builder.initial_cargo);
+ cargo
+ .arg("doc")
+ .arg("--target-dir")
+ .arg(&out_dir)
+ .env("RUSTC_BOOTSTRAP", "1")
+ .env("RUSTDOC", builder.rustdoc(self.compiler))
+ .env("RUSTC", builder.rustc(self.compiler))
+ .current_dir(path);
+ // FIXME: implement a `// compile-flags` command or similar
+ // instead of hard-coding this test
+ if entry.file_name() == "link_to_definition" {
+ cargo.env("RUSTDOCFLAGS", "-Zunstable-options --generate-link-to-definition");
+ }
+ builder.run(&mut cargo);
+ }
+ }
+
+ // We now run GUI tests.
+ let mut command = Command::new(&nodejs);
+ command
+ .arg(builder.build.src.join("src/tools/rustdoc-gui/tester.js"))
+ .arg("--jobs")
+ .arg(&builder.jobs().to_string())
+ .arg("--doc-folder")
+ .arg(out_dir.join("doc"))
+ .arg("--tests-folder")
+ .arg(builder.build.src.join("src/test/rustdoc-gui"));
+ for path in &builder.paths {
+ if let Some(p) = util::is_valid_test_suite_arg(path, "src/test/rustdoc-gui", builder) {
+ if !p.ends_with(".goml") {
+ eprintln!("A non-goml file was given: `{}`", path.display());
+ panic!("Cannot run rustdoc-gui tests");
+ }
+ if let Some(name) = path.file_name().and_then(|f| f.to_str()) {
+ command.arg("--file").arg(name);
+ }
+ }
+ }
+ for test_arg in builder.config.cmd.test_args() {
+ command.arg(test_arg);
+ }
+ builder.run(&mut command);
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct Tidy;
+
+impl Step for Tidy {
+ type Output = ();
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ /// Runs the `tidy` tool.
+ ///
+ /// This tool in `src/tools` checks up on various bits and pieces of style and
+ /// otherwise just implements a few lint-like checks that are specific to the
+ /// compiler itself.
+ ///
+ /// Once tidy passes, this step also runs `fmt --check` if tests are being run
+ /// for the `dev` or `nightly` channels.
+ fn run(self, builder: &Builder<'_>) {
+ let mut cmd = builder.tool_cmd(Tool::Tidy);
+ cmd.arg(&builder.src);
+ cmd.arg(&builder.initial_cargo);
+ cmd.arg(&builder.out);
+ cmd.arg(builder.jobs().to_string());
+ if builder.is_verbose() {
+ cmd.arg("--verbose");
+ }
+
+ builder.info("tidy check");
+ try_run(builder, &mut cmd);
+
+ if builder.config.channel == "dev" || builder.config.channel == "nightly" {
+ builder.info("fmt check");
+ if builder.initial_rustfmt().is_none() {
+ let inferred_rustfmt_dir = builder.config.initial_rustc.parent().unwrap();
+ eprintln!(
+ "\
+error: no `rustfmt` binary found in {PATH}
+info: `rust.channel` is currently set to \"{CHAN}\"
+help: if you are testing a beta branch, set `rust.channel` to \"beta\" in the `config.toml` file
+help: to skip test's attempt to check tidiness, pass `--exclude src/tools/tidy` to `x.py test`",
+ PATH = inferred_rustfmt_dir.display(),
+ CHAN = builder.config.channel,
+ );
+ crate::detail_exit(1);
+ }
+ crate::format::format(&builder, !builder.config.cmd.bless(), &[]);
+ }
+ }
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/tools/tidy")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Tidy);
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct ExpandYamlAnchors;
+
+impl Step for ExpandYamlAnchors {
+ type Output = ();
+ const ONLY_HOSTS: bool = true;
+
+ /// Ensure the `generate-ci-config` tool was run locally.
+ ///
+ /// The tool in `src/tools` reads the CI definition in `src/ci/builders.yml` and generates the
+ /// appropriate configuration for all our CI providers. This step ensures the tool was called
+ /// by the user before committing CI changes.
+ fn run(self, builder: &Builder<'_>) {
+ builder.info("Ensuring the YAML anchors in the GitHub Actions config were expanded");
+ try_run(
+ builder,
+ &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("check").arg(&builder.src),
+ );
+ }
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/tools/expand-yaml-anchors")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(ExpandYamlAnchors);
+ }
+}
+
+fn testdir(builder: &Builder<'_>, host: TargetSelection) -> PathBuf {
+ builder.out.join(host.triple).join("test")
+}
+
+macro_rules! default_test {
+ ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr }) => {
+ test!($name { path: $path, mode: $mode, suite: $suite, default: true, host: false });
+ };
+}
+
+macro_rules! default_test_with_compare_mode {
+ ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr,
+ compare_mode: $compare_mode:expr }) => {
+ test_with_compare_mode!($name {
+ path: $path,
+ mode: $mode,
+ suite: $suite,
+ default: true,
+ host: false,
+ compare_mode: $compare_mode
+ });
+ };
+}
+
+macro_rules! host_test {
+ ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr }) => {
+ test!($name { path: $path, mode: $mode, suite: $suite, default: true, host: true });
+ };
+}
+
+macro_rules! test {
+ ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, default: $default:expr,
+ host: $host:expr }) => {
+ test_definitions!($name {
+ path: $path,
+ mode: $mode,
+ suite: $suite,
+ default: $default,
+ host: $host,
+ compare_mode: None
+ });
+ };
+}
+
+macro_rules! test_with_compare_mode {
+ ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, default: $default:expr,
+ host: $host:expr, compare_mode: $compare_mode:expr }) => {
+ test_definitions!($name {
+ path: $path,
+ mode: $mode,
+ suite: $suite,
+ default: $default,
+ host: $host,
+ compare_mode: Some($compare_mode)
+ });
+ };
+}
+
+macro_rules! test_definitions {
+ ($name:ident {
+ path: $path:expr,
+ mode: $mode:expr,
+ suite: $suite:expr,
+ default: $default:expr,
+ host: $host:expr,
+ compare_mode: $compare_mode:expr
+ }) => {
+ #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+ pub struct $name {
+ pub compiler: Compiler,
+ pub target: TargetSelection,
+ }
+
+ impl Step for $name {
+ type Output = ();
+ const DEFAULT: bool = $default;
+ const ONLY_HOSTS: bool = $host;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.suite_path($path)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple());
+
+ run.builder.ensure($name { compiler, target: run.target });
+ }
+
+ fn run(self, builder: &Builder<'_>) {
+ builder.ensure(Compiletest {
+ compiler: self.compiler,
+ target: self.target,
+ mode: $mode,
+ suite: $suite,
+ path: $path,
+ compare_mode: $compare_mode,
+ })
+ }
+ }
+ };
+}
+
+default_test!(Ui { path: "src/test/ui", mode: "ui", suite: "ui" });
+
+default_test!(RunPassValgrind {
+ path: "src/test/run-pass-valgrind",
+ mode: "run-pass-valgrind",
+ suite: "run-pass-valgrind"
+});
+
+default_test!(MirOpt { path: "src/test/mir-opt", mode: "mir-opt", suite: "mir-opt" });
+
+default_test!(Codegen { path: "src/test/codegen", mode: "codegen", suite: "codegen" });
+
+default_test!(CodegenUnits {
+ path: "src/test/codegen-units",
+ mode: "codegen-units",
+ suite: "codegen-units"
+});
+
+default_test!(Incremental {
+ path: "src/test/incremental",
+ mode: "incremental",
+ suite: "incremental"
+});
+
+default_test_with_compare_mode!(Debuginfo {
+ path: "src/test/debuginfo",
+ mode: "debuginfo",
+ suite: "debuginfo",
+ compare_mode: "split-dwarf"
+});
+
+host_test!(UiFullDeps { path: "src/test/ui-fulldeps", mode: "ui", suite: "ui-fulldeps" });
+
+host_test!(Rustdoc { path: "src/test/rustdoc", mode: "rustdoc", suite: "rustdoc" });
+host_test!(RustdocUi { path: "src/test/rustdoc-ui", mode: "ui", suite: "rustdoc-ui" });
+
+host_test!(RustdocJson {
+ path: "src/test/rustdoc-json",
+ mode: "rustdoc-json",
+ suite: "rustdoc-json"
+});
+
+host_test!(Pretty { path: "src/test/pretty", mode: "pretty", suite: "pretty" });
+
+default_test!(RunMake { path: "src/test/run-make", mode: "run-make", suite: "run-make" });
+
+host_test!(RunMakeFullDeps {
+ path: "src/test/run-make-fulldeps",
+ mode: "run-make",
+ suite: "run-make-fulldeps"
+});
+
+default_test!(Assembly { path: "src/test/assembly", mode: "assembly", suite: "assembly" });
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+struct Compiletest {
+ compiler: Compiler,
+ target: TargetSelection,
+ mode: &'static str,
+ suite: &'static str,
+ path: &'static str,
+ compare_mode: Option<&'static str>,
+}
+
+impl Step for Compiletest {
+ type Output = ();
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.never()
+ }
+
+ /// Executes the `compiletest` tool to run a suite of tests.
+ ///
+ /// Compiles all tests with `compiler` for `target` with the specified
+ /// compiletest `mode` and `suite` arguments. For example `mode` can be
+ /// "run-pass" or `suite` can be something like `debuginfo`.
+ fn run(self, builder: &Builder<'_>) {
+ if builder.top_stage == 0 && env::var("COMPILETEST_FORCE_STAGE0").is_err() {
+ eprintln!("\
+error: `--stage 0` runs compiletest on the beta compiler, not your local changes, and will almost always cause tests to fail
+help: to test the compiler, use `--stage 1` instead
+help: to test the standard library, use `--stage 0 library/std` instead
+note: if you're sure you want to do this, please open an issue as to why. In the meantime, you can override this with `COMPILETEST_FORCE_STAGE0=1`."
+ );
+ crate::detail_exit(1);
+ }
+
+ let compiler = self.compiler;
+ let target = self.target;
+ let mode = self.mode;
+ let suite = self.suite;
+
+ // Path for test suite
+ let suite_path = self.path;
+
+ // Skip codegen tests if they aren't enabled in configuration.
+ if !builder.config.codegen_tests && suite == "codegen" {
+ return;
+ }
+
+ if suite == "debuginfo" {
+ builder
+ .ensure(dist::DebuggerScripts { sysroot: builder.sysroot(compiler), host: target });
+ }
+
+ if suite.ends_with("fulldeps") {
+ builder.ensure(compile::Rustc::new(compiler, target));
+ }
+
+ builder.ensure(compile::Std::new(compiler, target));
+ // ensure that `libproc_macro` is available on the host.
+ builder.ensure(compile::Std::new(compiler, compiler.host));
+
+ // Also provide `rust_test_helpers` for the host.
+ builder.ensure(native::TestHelpers { target: compiler.host });
+
+ // As well as the target, except for plain wasm32, which can't build it
+ if !target.contains("wasm") || target.contains("emscripten") {
+ builder.ensure(native::TestHelpers { target });
+ }
+
+ builder.ensure(RemoteCopyLibs { compiler, target });
+
+ let mut cmd = builder.tool_cmd(Tool::Compiletest);
+
+ // compiletest currently has... a lot of arguments, so let's just pass all
+ // of them!
+
+ cmd.arg("--compile-lib-path").arg(builder.rustc_libdir(compiler));
+ cmd.arg("--run-lib-path").arg(builder.sysroot_libdir(compiler, target));
+ cmd.arg("--rustc-path").arg(builder.rustc(compiler));
+
+ let is_rustdoc = suite.ends_with("rustdoc-ui") || suite.ends_with("rustdoc-js");
+
+ // Avoid depending on rustdoc when we don't need it.
+ if mode == "rustdoc"
+ || mode == "run-make"
+ || (mode == "ui" && is_rustdoc)
+ || mode == "js-doc-test"
+ || mode == "rustdoc-json"
+ {
+ cmd.arg("--rustdoc-path").arg(builder.rustdoc(compiler));
+ }
+
+ if mode == "rustdoc-json" {
+ // Use the beta compiler for jsondocck
+ let json_compiler = compiler.with_stage(0);
+ cmd.arg("--jsondocck-path")
+ .arg(builder.ensure(tool::JsonDocCk { compiler: json_compiler, target }));
+ }
+
+ if mode == "run-make" {
+ let rust_demangler = builder
+ .ensure(tool::RustDemangler {
+ compiler,
+ target: compiler.host,
+ extra_features: Vec::new(),
+ })
+ .expect("in-tree tool");
+ cmd.arg("--rust-demangler-path").arg(rust_demangler);
+ }
+
+ cmd.arg("--src-base").arg(builder.src.join("src/test").join(suite));
+ cmd.arg("--build-base").arg(testdir(builder, compiler.host).join(suite));
+ cmd.arg("--stage-id").arg(format!("stage{}-{}", compiler.stage, target));
+ cmd.arg("--suite").arg(suite);
+ cmd.arg("--mode").arg(mode);
+ cmd.arg("--target").arg(target.rustc_target_arg());
+ cmd.arg("--host").arg(&*compiler.host.triple);
+ cmd.arg("--llvm-filecheck").arg(builder.llvm_filecheck(builder.config.build));
+
+ if builder.config.cmd.bless() {
+ cmd.arg("--bless");
+ }
+
+ if builder.config.cmd.force_rerun() {
+ cmd.arg("--force-rerun");
+ }
+
+ let compare_mode =
+ builder.config.cmd.compare_mode().or_else(|| {
+ if builder.config.test_compare_mode { self.compare_mode } else { None }
+ });
+
+ if let Some(ref pass) = builder.config.cmd.pass() {
+ cmd.arg("--pass");
+ cmd.arg(pass);
+ }
+
+ if let Some(ref run) = builder.config.cmd.run() {
+ cmd.arg("--run");
+ cmd.arg(run);
+ }
+
+ if let Some(ref nodejs) = builder.config.nodejs {
+ cmd.arg("--nodejs").arg(nodejs);
+ }
+ if let Some(ref npm) = builder.config.npm {
+ cmd.arg("--npm").arg(npm);
+ }
+ if builder.config.rust_optimize_tests {
+ cmd.arg("--optimize-tests");
+ }
+ let mut flags = if is_rustdoc { Vec::new() } else { vec!["-Crpath".to_string()] };
+ flags.push(format!("-Cdebuginfo={}", builder.config.rust_debuginfo_level_tests));
+ flags.push(builder.config.cmd.rustc_args().join(" "));
+
+ if let Some(linker) = builder.linker(target) {
+ cmd.arg("--linker").arg(linker);
+ }
+
+ let mut hostflags = flags.clone();
+ hostflags.push(format!("-Lnative={}", builder.test_helpers_out(compiler.host).display()));
+ hostflags.extend(builder.lld_flags(compiler.host));
+ cmd.arg("--host-rustcflags").arg(hostflags.join(" "));
+
+ let mut targetflags = flags;
+ targetflags.push(format!("-Lnative={}", builder.test_helpers_out(target).display()));
+ targetflags.extend(builder.lld_flags(target));
+ cmd.arg("--target-rustcflags").arg(targetflags.join(" "));
+
+ cmd.arg("--python").arg(builder.python());
+
+ if let Some(ref gdb) = builder.config.gdb {
+ cmd.arg("--gdb").arg(gdb);
+ }
+
+ let run = |cmd: &mut Command| {
+ cmd.output().map(|output| {
+ String::from_utf8_lossy(&output.stdout)
+ .lines()
+ .next()
+ .unwrap_or_else(|| panic!("{:?} failed {:?}", cmd, output))
+ .to_string()
+ })
+ };
+ let lldb_exe = "lldb";
+ let lldb_version = Command::new(lldb_exe)
+ .arg("--version")
+ .output()
+ .map(|output| String::from_utf8_lossy(&output.stdout).to_string())
+ .ok();
+ if let Some(ref vers) = lldb_version {
+ cmd.arg("--lldb-version").arg(vers);
+ let lldb_python_dir = run(Command::new(lldb_exe).arg("-P")).ok();
+ if let Some(ref dir) = lldb_python_dir {
+ cmd.arg("--lldb-python-dir").arg(dir);
+ }
+ }
+
+ if util::forcing_clang_based_tests() {
+ let clang_exe = builder.llvm_out(target).join("bin").join("clang");
+ cmd.arg("--run-clang-based-tests-with").arg(clang_exe);
+ }
+
+ // Get paths from cmd args
+ let paths = match &builder.config.cmd {
+ Subcommand::Test { ref paths, .. } => &paths[..],
+ _ => &[],
+ };
+
+ // Get test-args by striping suite path
+ let mut test_args: Vec<&str> = paths
+ .iter()
+ .filter_map(|p| util::is_valid_test_suite_arg(p, suite_path, builder))
+ .collect();
+
+ test_args.append(&mut builder.config.cmd.test_args());
+
+ cmd.args(&test_args);
+
+ if builder.is_verbose() {
+ cmd.arg("--verbose");
+ }
+
+ if !builder.config.verbose_tests {
+ cmd.arg("--quiet");
+ }
+
+ let mut llvm_components_passed = false;
+ let mut copts_passed = false;
+ if builder.config.llvm_enabled() {
+ let llvm_config = builder.ensure(native::Llvm { target: builder.config.build });
+ if !builder.config.dry_run {
+ let llvm_version = output(Command::new(&llvm_config).arg("--version"));
+ let llvm_components = output(Command::new(&llvm_config).arg("--components"));
+ // Remove trailing newline from llvm-config output.
+ cmd.arg("--llvm-version")
+ .arg(llvm_version.trim())
+ .arg("--llvm-components")
+ .arg(llvm_components.trim());
+ llvm_components_passed = true;
+ }
+ if !builder.is_rust_llvm(target) {
+ cmd.arg("--system-llvm");
+ }
+
+ // Tests that use compiler libraries may inherit the `-lLLVM` link
+ // requirement, but the `-L` library path is not propagated across
+ // separate compilations. We can add LLVM's library path to the
+ // platform-specific environment variable as a workaround.
+ if !builder.config.dry_run && suite.ends_with("fulldeps") {
+ let llvm_libdir = output(Command::new(&llvm_config).arg("--libdir"));
+ add_link_lib_path(vec![llvm_libdir.trim().into()], &mut cmd);
+ }
+
+ // Only pass correct values for these flags for the `run-make` suite as it
+ // requires that a C++ compiler was configured which isn't always the case.
+ if !builder.config.dry_run && matches!(suite, "run-make" | "run-make-fulldeps") {
+ // The llvm/bin directory contains many useful cross-platform
+ // tools. Pass the path to run-make tests so they can use them.
+ let llvm_bin_path = llvm_config
+ .parent()
+ .expect("Expected llvm-config to be contained in directory");
+ assert!(llvm_bin_path.is_dir());
+ cmd.arg("--llvm-bin-dir").arg(llvm_bin_path);
+
+ // If LLD is available, add it to the PATH
+ if builder.config.lld_enabled {
+ let lld_install_root =
+ builder.ensure(native::Lld { target: builder.config.build });
+
+ let lld_bin_path = lld_install_root.join("bin");
+
+ let old_path = env::var_os("PATH").unwrap_or_default();
+ let new_path = env::join_paths(
+ std::iter::once(lld_bin_path).chain(env::split_paths(&old_path)),
+ )
+ .expect("Could not add LLD bin path to PATH");
+ cmd.env("PATH", new_path);
+ }
+ }
+ }
+
+ // Only pass correct values for these flags for the `run-make` suite as it
+ // requires that a C++ compiler was configured which isn't always the case.
+ if !builder.config.dry_run && matches!(suite, "run-make" | "run-make-fulldeps") {
+ cmd.arg("--cc")
+ .arg(builder.cc(target))
+ .arg("--cxx")
+ .arg(builder.cxx(target).unwrap())
+ .arg("--cflags")
+ .arg(builder.cflags(target, GitRepo::Rustc, CLang::C).join(" "))
+ .arg("--cxxflags")
+ .arg(builder.cflags(target, GitRepo::Rustc, CLang::Cxx).join(" "));
+ copts_passed = true;
+ if let Some(ar) = builder.ar(target) {
+ cmd.arg("--ar").arg(ar);
+ }
+ }
+
+ if !llvm_components_passed {
+ cmd.arg("--llvm-components").arg("");
+ }
+ if !copts_passed {
+ cmd.arg("--cc")
+ .arg("")
+ .arg("--cxx")
+ .arg("")
+ .arg("--cflags")
+ .arg("")
+ .arg("--cxxflags")
+ .arg("");
+ }
+
+ if builder.remote_tested(target) {
+ cmd.arg("--remote-test-client").arg(builder.tool_exe(Tool::RemoteTestClient));
+ }
+
+ // Running a C compiler on MSVC requires a few env vars to be set, to be
+ // sure to set them here.
+ //
+ // Note that if we encounter `PATH` we make sure to append to our own `PATH`
+ // rather than stomp over it.
+ if target.contains("msvc") {
+ for &(ref k, ref v) in builder.cc[&target].env() {
+ if k != "PATH" {
+ cmd.env(k, v);
+ }
+ }
+ }
+ cmd.env("RUSTC_BOOTSTRAP", "1");
+ // Override the rustc version used in symbol hashes to reduce the amount of normalization
+ // needed when diffing test output.
+ cmd.env("RUSTC_FORCE_RUSTC_VERSION", "compiletest");
+ cmd.env("DOC_RUST_LANG_ORG_CHANNEL", builder.doc_rust_lang_org_channel());
+ builder.add_rust_test_threads(&mut cmd);
+
+ if builder.config.sanitizers_enabled(target) {
+ cmd.env("RUSTC_SANITIZER_SUPPORT", "1");
+ }
+
+ if builder.config.profiler_enabled(target) {
+ cmd.env("RUSTC_PROFILER_SUPPORT", "1");
+ }
+
+ cmd.env("RUST_TEST_TMPDIR", builder.tempdir());
+
+ cmd.arg("--adb-path").arg("adb");
+ cmd.arg("--adb-test-dir").arg(ADB_TEST_DIR);
+ if target.contains("android") {
+ // Assume that cc for this target comes from the android sysroot
+ cmd.arg("--android-cross-path")
+ .arg(builder.cc(target).parent().unwrap().parent().unwrap());
+ } else {
+ cmd.arg("--android-cross-path").arg("");
+ }
+
+ if builder.config.cmd.rustfix_coverage() {
+ cmd.arg("--rustfix-coverage");
+ }
+
+ cmd.env("BOOTSTRAP_CARGO", &builder.initial_cargo);
+
+ cmd.arg("--channel").arg(&builder.config.channel);
+
+ builder.ci_env.force_coloring_in_ci(&mut cmd);
+
+ builder.info(&format!(
+ "Check compiletest suite={} mode={} ({} -> {})",
+ suite, mode, &compiler.host, target
+ ));
+ let _time = util::timeit(&builder);
+ try_run(builder, &mut cmd);
+
+ if let Some(compare_mode) = compare_mode {
+ cmd.arg("--compare-mode").arg(compare_mode);
+ builder.info(&format!(
+ "Check compiletest suite={} mode={} compare_mode={} ({} -> {})",
+ suite, mode, compare_mode, &compiler.host, target
+ ));
+ let _time = util::timeit(&builder);
+ try_run(builder, &mut cmd);
+ }
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+struct BookTest {
+ compiler: Compiler,
+ path: PathBuf,
+ name: &'static str,
+ is_ext_doc: bool,
+}
+
+impl Step for BookTest {
+ type Output = ();
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.never()
+ }
+
+ /// Runs the documentation tests for a book in `src/doc`.
+ ///
+ /// This uses the `rustdoc` that sits next to `compiler`.
+ fn run(self, builder: &Builder<'_>) {
+ // External docs are different from local because:
+ // - Some books need pre-processing by mdbook before being tested.
+ // - They need to save their state to toolstate.
+ // - They are only tested on the "checktools" builders.
+ //
+ // The local docs are tested by default, and we don't want to pay the
+ // cost of building mdbook, so they use `rustdoc --test` directly.
+ // Also, the unstable book is special because SUMMARY.md is generated,
+ // so it is easier to just run `rustdoc` on its files.
+ if self.is_ext_doc {
+ self.run_ext_doc(builder);
+ } else {
+ self.run_local_doc(builder);
+ }
+ }
+}
+
+impl BookTest {
+ /// This runs the equivalent of `mdbook test` (via the rustbook wrapper)
+ /// which in turn runs `rustdoc --test` on each file in the book.
+ fn run_ext_doc(self, builder: &Builder<'_>) {
+ let compiler = self.compiler;
+
+ builder.ensure(compile::Std::new(compiler, compiler.host));
+
+ // mdbook just executes a binary named "rustdoc", so we need to update
+ // PATH so that it points to our rustdoc.
+ let mut rustdoc_path = builder.rustdoc(compiler);
+ rustdoc_path.pop();
+ let old_path = env::var_os("PATH").unwrap_or_default();
+ let new_path = env::join_paths(iter::once(rustdoc_path).chain(env::split_paths(&old_path)))
+ .expect("could not add rustdoc to PATH");
+
+ let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook);
+ let path = builder.src.join(&self.path);
+ // Books often have feature-gated example text.
+ rustbook_cmd.env("RUSTC_BOOTSTRAP", "1");
+ rustbook_cmd.env("PATH", new_path).arg("test").arg(path);
+ builder.add_rust_test_threads(&mut rustbook_cmd);
+ builder.info(&format!("Testing rustbook {}", self.path.display()));
+ let _time = util::timeit(&builder);
+ let toolstate = if try_run(builder, &mut rustbook_cmd) {
+ ToolState::TestPass
+ } else {
+ ToolState::TestFail
+ };
+ builder.save_toolstate(self.name, toolstate);
+ }
+
+ /// This runs `rustdoc --test` on all `.md` files in the path.
+ fn run_local_doc(self, builder: &Builder<'_>) {
+ let compiler = self.compiler;
+
+ builder.ensure(compile::Std::new(compiler, compiler.host));
+
+ // Do a breadth-first traversal of the `src/doc` directory and just run
+ // tests for all files that end in `*.md`
+ let mut stack = vec![builder.src.join(self.path)];
+ let _time = util::timeit(&builder);
+ let mut files = Vec::new();
+ while let Some(p) = stack.pop() {
+ if p.is_dir() {
+ stack.extend(t!(p.read_dir()).map(|p| t!(p).path()));
+ continue;
+ }
+
+ if p.extension().and_then(|s| s.to_str()) != Some("md") {
+ continue;
+ }
+
+ files.push(p);
+ }
+
+ files.sort();
+
+ for file in files {
+ markdown_test(builder, compiler, &file);
+ }
+ }
+}
+
+macro_rules! test_book {
+ ($($name:ident, $path:expr, $book_name:expr, default=$default:expr;)+) => {
+ $(
+ #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+ pub struct $name {
+ compiler: Compiler,
+ }
+
+ impl Step for $name {
+ type Output = ();
+ const DEFAULT: bool = $default;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path($path)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure($name {
+ compiler: run.builder.compiler(run.builder.top_stage, run.target),
+ });
+ }
+
+ fn run(self, builder: &Builder<'_>) {
+ builder.ensure(BookTest {
+ compiler: self.compiler,
+ path: PathBuf::from($path),
+ name: $book_name,
+ is_ext_doc: !$default,
+ });
+ }
+ }
+ )+
+ }
+}
+
+test_book!(
+ Nomicon, "src/doc/nomicon", "nomicon", default=false;
+ Reference, "src/doc/reference", "reference", default=false;
+ RustdocBook, "src/doc/rustdoc", "rustdoc", default=true;
+ RustcBook, "src/doc/rustc", "rustc", default=true;
+ RustByExample, "src/doc/rust-by-example", "rust-by-example", default=false;
+ EmbeddedBook, "src/doc/embedded-book", "embedded-book", default=false;
+ TheBook, "src/doc/book", "book", default=false;
+ UnstableBook, "src/doc/unstable-book", "unstable-book", default=true;
+ EditionGuide, "src/doc/edition-guide", "edition-guide", default=false;
+);
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct ErrorIndex {
+ compiler: Compiler,
+}
+
+impl Step for ErrorIndex {
+ type Output = ();
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/tools/error_index_generator")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ // error_index_generator depends on librustdoc. Use the compiler that
+ // is normally used to build rustdoc for other tests (like compiletest
+ // tests in src/test/rustdoc) so that it shares the same artifacts.
+ let compiler = run.builder.compiler(run.builder.top_stage, run.builder.config.build);
+ run.builder.ensure(ErrorIndex { compiler });
+ }
+
+ /// Runs the error index generator tool to execute the tests located in the error
+ /// index.
+ ///
+ /// The `error_index_generator` tool lives in `src/tools` and is used to
+ /// generate a markdown file from the error indexes of the code base which is
+ /// then passed to `rustdoc --test`.
+ fn run(self, builder: &Builder<'_>) {
+ let compiler = self.compiler;
+
+ let dir = testdir(builder, compiler.host);
+ t!(fs::create_dir_all(&dir));
+ let output = dir.join("error-index.md");
+
+ let mut tool = tool::ErrorIndex::command(builder);
+ tool.arg("markdown").arg(&output);
+
+ builder.info(&format!("Testing error-index stage{}", compiler.stage));
+ let _time = util::timeit(&builder);
+ builder.run_quiet(&mut tool);
+ // The tests themselves need to link to std, so make sure it is
+ // available.
+ builder.ensure(compile::Std::new(compiler, compiler.host));
+ markdown_test(builder, compiler, &output);
+ }
+}
+
+fn markdown_test(builder: &Builder<'_>, compiler: Compiler, markdown: &Path) -> bool {
+ if let Ok(contents) = fs::read_to_string(markdown) {
+ if !contents.contains("```") {
+ return true;
+ }
+ }
+
+ builder.info(&format!("doc tests for: {}", markdown.display()));
+ let mut cmd = builder.rustdoc_cmd(compiler);
+ builder.add_rust_test_threads(&mut cmd);
+ // allow for unstable options such as new editions
+ cmd.arg("-Z");
+ cmd.arg("unstable-options");
+ cmd.arg("--test");
+ cmd.arg(markdown);
+ cmd.env("RUSTC_BOOTSTRAP", "1");
+
+ let test_args = builder.config.cmd.test_args().join(" ");
+ cmd.arg("--test-args").arg(test_args);
+
+ if builder.config.verbose_tests {
+ try_run(builder, &mut cmd)
+ } else {
+ try_run_quiet(builder, &mut cmd)
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct RustcGuide;
+
+impl Step for RustcGuide {
+ type Output = ();
+ const DEFAULT: bool = false;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/doc/rustc-dev-guide")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(RustcGuide);
+ }
+
+ fn run(self, builder: &Builder<'_>) {
+ let relative_path = Path::new("src").join("doc").join("rustc-dev-guide");
+ builder.update_submodule(&relative_path);
+
+ let src = builder.src.join(relative_path);
+ let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook);
+ let toolstate = if try_run(builder, rustbook_cmd.arg("linkcheck").arg(&src)) {
+ ToolState::TestPass
+ } else {
+ ToolState::TestFail
+ };
+ builder.save_toolstate("rustc-dev-guide", toolstate);
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct CrateLibrustc {
+ compiler: Compiler,
+ target: TargetSelection,
+ test_kind: TestKind,
+ crates: Vec<Interned<String>>,
+}
+
+impl Step for CrateLibrustc {
+ type Output = ();
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.crate_or_deps("rustc-main")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ let builder = run.builder;
+ let host = run.build_triple();
+ let compiler = builder.compiler_for(builder.top_stage, host, host);
+ let crates = run
+ .paths
+ .iter()
+ .map(|p| builder.crate_paths[&p.assert_single_path().path].clone())
+ .collect();
+ let test_kind = builder.kind.into();
+
+ builder.ensure(CrateLibrustc { compiler, target: run.target, test_kind, crates });
+ }
+
+ fn run(self, builder: &Builder<'_>) {
+ builder.ensure(Crate {
+ compiler: self.compiler,
+ target: self.target,
+ mode: Mode::Rustc,
+ test_kind: self.test_kind,
+ crates: self.crates,
+ });
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct Crate {
+ pub compiler: Compiler,
+ pub target: TargetSelection,
+ pub mode: Mode,
+ pub test_kind: TestKind,
+ pub crates: Vec<Interned<String>>,
+}
+
+impl Step for Crate {
+ type Output = ();
+ const DEFAULT: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.crate_or_deps("test")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ let builder = run.builder;
+ let host = run.build_triple();
+ let compiler = builder.compiler_for(builder.top_stage, host, host);
+ let test_kind = builder.kind.into();
+ let crates = run
+ .paths
+ .iter()
+ .map(|p| builder.crate_paths[&p.assert_single_path().path].clone())
+ .collect();
+
+ builder.ensure(Crate { compiler, target: run.target, mode: Mode::Std, test_kind, crates });
+ }
+
+ /// Runs all unit tests plus documentation tests for a given crate defined
+ /// by a `Cargo.toml` (single manifest)
+ ///
+ /// This is what runs tests for crates like the standard library, compiler, etc.
+ /// It essentially is the driver for running `cargo test`.
+ ///
+ /// Currently this runs all tests for a DAG by passing a bunch of `-p foo`
+ /// arguments, and those arguments are discovered from `cargo metadata`.
+ fn run(self, builder: &Builder<'_>) {
+ let compiler = self.compiler;
+ let target = self.target;
+ let mode = self.mode;
+ let test_kind = self.test_kind;
+
+ builder.ensure(compile::Std::new(compiler, target));
+ builder.ensure(RemoteCopyLibs { compiler, target });
+
+ // If we're not doing a full bootstrap but we're testing a stage2
+ // version of libstd, then what we're actually testing is the libstd
+ // produced in stage1. Reflect that here by updating the compiler that
+ // we're working with automatically.
+ let compiler = builder.compiler_for(compiler.stage, compiler.host, target);
+
+ let mut cargo =
+ builder.cargo(compiler, mode, SourceType::InTree, target, test_kind.subcommand());
+ match mode {
+ Mode::Std => {
+ compile::std_cargo(builder, target, compiler.stage, &mut cargo);
+ }
+ Mode::Rustc => {
+ compile::rustc_cargo(builder, &mut cargo, target);
+ }
+ _ => panic!("can only test libraries"),
+ };
+
+ // Build up the base `cargo test` command.
+ //
+ // Pass in some standard flags then iterate over the graph we've discovered
+ // in `cargo metadata` with the maps above and figure out what `-p`
+ // arguments need to get passed.
+ if test_kind.subcommand() == "test" && !builder.fail_fast {
+ cargo.arg("--no-fail-fast");
+ }
+ match builder.doc_tests {
+ DocTests::Only => {
+ cargo.arg("--doc");
+ }
+ DocTests::No => {
+ cargo.args(&["--lib", "--bins", "--examples", "--tests", "--benches"]);
+ }
+ DocTests::Yes => {}
+ }
+
+ for krate in &self.crates {
+ cargo.arg("-p").arg(krate);
+ }
+
+ // The tests are going to run with the *target* libraries, so we need to
+ // ensure that those libraries show up in the LD_LIBRARY_PATH equivalent.
+ //
+ // Note that to run the compiler we need to run with the *host* libraries,
+ // but our wrapper scripts arrange for that to be the case anyway.
+ let mut dylib_path = dylib_path();
+ dylib_path.insert(0, PathBuf::from(&*builder.sysroot_libdir(compiler, target)));
+ cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap());
+
+ cargo.arg("--");
+ cargo.args(&builder.config.cmd.test_args());
+
+ if !builder.config.verbose_tests {
+ cargo.arg("--quiet");
+ }
+
+ if target.contains("emscripten") {
+ cargo.env(
+ format!("CARGO_TARGET_{}_RUNNER", envify(&target.triple)),
+ builder.config.nodejs.as_ref().expect("nodejs not configured"),
+ );
+ } else if target.starts_with("wasm32") {
+ let node = builder.config.nodejs.as_ref().expect("nodejs not configured");
+ let runner =
+ format!("{} {}/src/etc/wasm32-shim.js", node.display(), builder.src.display());
+ cargo.env(format!("CARGO_TARGET_{}_RUNNER", envify(&target.triple)), &runner);
+ } else if builder.remote_tested(target) {
+ cargo.env(
+ format!("CARGO_TARGET_{}_RUNNER", envify(&target.triple)),
+ format!("{} run 0", builder.tool_exe(Tool::RemoteTestClient).display()),
+ );
+ }
+
+ builder.info(&format!(
+ "{} {:?} stage{} ({} -> {})",
+ test_kind, self.crates, compiler.stage, &compiler.host, target
+ ));
+ let _time = util::timeit(&builder);
+ try_run(builder, &mut cargo.into());
+ }
+}
+
+/// Rustdoc is special in various ways, which is why this step is different from `Crate`.
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct CrateRustdoc {
+ host: TargetSelection,
+ test_kind: TestKind,
+}
+
+impl Step for CrateRustdoc {
+ type Output = ();
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.paths(&["src/librustdoc", "src/tools/rustdoc"])
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ let builder = run.builder;
+
+ let test_kind = builder.kind.into();
+
+ builder.ensure(CrateRustdoc { host: run.target, test_kind });
+ }
+
+ fn run(self, builder: &Builder<'_>) {
+ let test_kind = self.test_kind;
+ let target = self.host;
+
+ let compiler = if builder.download_rustc() {
+ builder.compiler(builder.top_stage, target)
+ } else {
+ // Use the previous stage compiler to reuse the artifacts that are
+ // created when running compiletest for src/test/rustdoc. If this used
+ // `compiler`, then it would cause rustdoc to be built *again*, which
+ // isn't really necessary.
+ builder.compiler_for(builder.top_stage, target, target)
+ };
+ builder.ensure(compile::Rustc::new(compiler, target));
+
+ let mut cargo = tool::prepare_tool_cargo(
+ builder,
+ compiler,
+ Mode::ToolRustc,
+ target,
+ test_kind.subcommand(),
+ "src/tools/rustdoc",
+ SourceType::InTree,
+ &[],
+ );
+ if test_kind.subcommand() == "test" && !builder.fail_fast {
+ cargo.arg("--no-fail-fast");
+ }
+ match builder.doc_tests {
+ DocTests::Only => {
+ cargo.arg("--doc");
+ }
+ DocTests::No => {
+ cargo.args(&["--lib", "--bins", "--examples", "--tests", "--benches"]);
+ }
+ DocTests::Yes => {}
+ }
+
+ cargo.arg("-p").arg("rustdoc:0.0.0");
+
+ cargo.arg("--");
+ cargo.args(&builder.config.cmd.test_args());
+
+ if self.host.contains("musl") {
+ cargo.arg("'-Ctarget-feature=-crt-static'");
+ }
+
+ // This is needed for running doctests on librustdoc. This is a bit of
+ // an unfortunate interaction with how bootstrap works and how cargo
+ // sets up the dylib path, and the fact that the doctest (in
+ // html/markdown.rs) links to rustc-private libs. For stage1, the
+ // compiler host dylibs (in stage1/lib) are not the same as the target
+ // dylibs (in stage1/lib/rustlib/...). This is different from a normal
+ // rust distribution where they are the same.
+ //
+ // On the cargo side, normal tests use `target_process` which handles
+ // setting up the dylib for a *target* (stage1/lib/rustlib/... in this
+ // case). However, for doctests it uses `rustdoc_process` which only
+ // sets up the dylib path for the *host* (stage1/lib), which is the
+ // wrong directory.
+ //
+ // Recall that we special-cased `compiler_for(top_stage)` above, so we always use stage1.
+ //
+ // It should be considered to just stop running doctests on
+ // librustdoc. There is only one test, and it doesn't look too
+ // important. There might be other ways to avoid this, but it seems
+ // pretty convoluted.
+ //
+ // See also https://github.com/rust-lang/rust/issues/13983 where the
+ // host vs target dylibs for rustdoc are consistently tricky to deal
+ // with.
+ //
+ // Note that this set the host libdir for `download_rustc`, which uses a normal rust distribution.
+ let libdir = if builder.download_rustc() {
+ builder.rustc_libdir(compiler)
+ } else {
+ builder.sysroot_libdir(compiler, target).to_path_buf()
+ };
+ let mut dylib_path = dylib_path();
+ dylib_path.insert(0, PathBuf::from(&*libdir));
+ cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap());
+
+ if !builder.config.verbose_tests {
+ cargo.arg("--quiet");
+ }
+
+ builder.info(&format!(
+ "{} rustdoc stage{} ({} -> {})",
+ test_kind, compiler.stage, &compiler.host, target
+ ));
+ let _time = util::timeit(&builder);
+
+ try_run(builder, &mut cargo.into());
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct CrateRustdocJsonTypes {
+ host: TargetSelection,
+ test_kind: TestKind,
+}
+
+impl Step for CrateRustdocJsonTypes {
+ type Output = ();
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/rustdoc-json-types")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ let builder = run.builder;
+
+ let test_kind = builder.kind.into();
+
+ builder.ensure(CrateRustdocJsonTypes { host: run.target, test_kind });
+ }
+
+ fn run(self, builder: &Builder<'_>) {
+ let test_kind = self.test_kind;
+ let target = self.host;
+
+ // Use the previous stage compiler to reuse the artifacts that are
+ // created when running compiletest for src/test/rustdoc. If this used
+ // `compiler`, then it would cause rustdoc to be built *again*, which
+ // isn't really necessary.
+ let compiler = builder.compiler_for(builder.top_stage, target, target);
+ builder.ensure(compile::Rustc::new(compiler, target));
+
+ let mut cargo = tool::prepare_tool_cargo(
+ builder,
+ compiler,
+ Mode::ToolRustc,
+ target,
+ test_kind.subcommand(),
+ "src/rustdoc-json-types",
+ SourceType::InTree,
+ &[],
+ );
+ if test_kind.subcommand() == "test" && !builder.fail_fast {
+ cargo.arg("--no-fail-fast");
+ }
+
+ cargo.arg("-p").arg("rustdoc-json-types");
+
+ cargo.arg("--");
+ cargo.args(&builder.config.cmd.test_args());
+
+ if self.host.contains("musl") {
+ cargo.arg("'-Ctarget-feature=-crt-static'");
+ }
+
+ if !builder.config.verbose_tests {
+ cargo.arg("--quiet");
+ }
+
+ builder.info(&format!(
+ "{} rustdoc-json-types stage{} ({} -> {})",
+ test_kind, compiler.stage, &compiler.host, target
+ ));
+ let _time = util::timeit(&builder);
+
+ try_run(builder, &mut cargo.into());
+ }
+}
+
+/// Some test suites are run inside emulators or on remote devices, and most
+/// of our test binaries are linked dynamically which means we need to ship
+/// the standard library and such to the emulator ahead of time. This step
+/// represents this and is a dependency of all test suites.
+///
+/// Most of the time this is a no-op. For some steps such as shipping data to
+/// QEMU we have to build our own tools so we've got conditional dependencies
+/// on those programs as well. Note that the remote test client is built for
+/// the build target (us) and the server is built for the target.
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct RemoteCopyLibs {
+ compiler: Compiler,
+ target: TargetSelection,
+}
+
+impl Step for RemoteCopyLibs {
+ type Output = ();
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.never()
+ }
+
+ fn run(self, builder: &Builder<'_>) {
+ let compiler = self.compiler;
+ let target = self.target;
+ if !builder.remote_tested(target) {
+ return;
+ }
+
+ builder.ensure(compile::Std::new(compiler, target));
+
+ builder.info(&format!("REMOTE copy libs to emulator ({})", target));
+
+ let server = builder.ensure(tool::RemoteTestServer { compiler, target });
+
+ // Spawn the emulator and wait for it to come online
+ let tool = builder.tool_exe(Tool::RemoteTestClient);
+ let mut cmd = Command::new(&tool);
+ cmd.arg("spawn-emulator").arg(target.triple).arg(&server).arg(builder.tempdir());
+ if let Some(rootfs) = builder.qemu_rootfs(target) {
+ cmd.arg(rootfs);
+ }
+ builder.run(&mut cmd);
+
+ // Push all our dylibs to the emulator
+ for f in t!(builder.sysroot_libdir(compiler, target).read_dir()) {
+ let f = t!(f);
+ let name = f.file_name().into_string().unwrap();
+ if util::is_dylib(&name) {
+ builder.run(Command::new(&tool).arg("push").arg(f.path()));
+ }
+ }
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct Distcheck;
+
+impl Step for Distcheck {
+ type Output = ();
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.alias("distcheck")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Distcheck);
+ }
+
+ /// Runs "distcheck", a 'make check' from a tarball
+ fn run(self, builder: &Builder<'_>) {
+ builder.info("Distcheck");
+ let dir = builder.tempdir().join("distcheck");
+ let _ = fs::remove_dir_all(&dir);
+ t!(fs::create_dir_all(&dir));
+
+ // Guarantee that these are built before we begin running.
+ builder.ensure(dist::PlainSourceTarball);
+ builder.ensure(dist::Src);
+
+ let mut cmd = Command::new("tar");
+ cmd.arg("-xf")
+ .arg(builder.ensure(dist::PlainSourceTarball).tarball())
+ .arg("--strip-components=1")
+ .current_dir(&dir);
+ builder.run(&mut cmd);
+ builder.run(
+ Command::new("./configure")
+ .args(&builder.config.configure_args)
+ .arg("--enable-vendor")
+ .current_dir(&dir),
+ );
+ builder.run(
+ Command::new(util::make(&builder.config.build.triple)).arg("check").current_dir(&dir),
+ );
+
+ // Now make sure that rust-src has all of libstd's dependencies
+ builder.info("Distcheck rust-src");
+ let dir = builder.tempdir().join("distcheck-src");
+ let _ = fs::remove_dir_all(&dir);
+ t!(fs::create_dir_all(&dir));
+
+ let mut cmd = Command::new("tar");
+ cmd.arg("-xf")
+ .arg(builder.ensure(dist::Src).tarball())
+ .arg("--strip-components=1")
+ .current_dir(&dir);
+ builder.run(&mut cmd);
+
+ let toml = dir.join("rust-src/lib/rustlib/src/rust/library/std/Cargo.toml");
+ builder.run(
+ Command::new(&builder.initial_cargo)
+ .arg("generate-lockfile")
+ .arg("--manifest-path")
+ .arg(&toml)
+ .current_dir(&dir),
+ );
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct Bootstrap;
+
+impl Step for Bootstrap {
+ type Output = ();
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ /// Tests the build system itself.
+ fn run(self, builder: &Builder<'_>) {
+ let mut check_bootstrap = Command::new(&builder.python());
+ check_bootstrap.arg("bootstrap_test.py").current_dir(builder.src.join("src/bootstrap/"));
+ try_run(builder, &mut check_bootstrap);
+
+ let mut cmd = Command::new(&builder.initial_cargo);
+ cmd.arg("test")
+ .current_dir(builder.src.join("src/bootstrap"))
+ .env("RUSTFLAGS", "-Cdebuginfo=2")
+ .env("CARGO_TARGET_DIR", builder.out.join("bootstrap"))
+ .env("RUSTC_BOOTSTRAP", "1")
+ .env("RUSTDOC", builder.rustdoc(builder.compiler(0, builder.build.build)))
+ .env("RUSTC", &builder.initial_rustc);
+ if let Some(flags) = option_env!("RUSTFLAGS") {
+ // Use the same rustc flags for testing as for "normal" compilation,
+ // so that Cargo doesn’t recompile the entire dependency graph every time:
+ // https://github.com/rust-lang/rust/issues/49215
+ cmd.env("RUSTFLAGS", flags);
+ }
+ if !builder.fail_fast {
+ cmd.arg("--no-fail-fast");
+ }
+ match builder.doc_tests {
+ DocTests::Only => {
+ cmd.arg("--doc");
+ }
+ DocTests::No => {
+ cmd.args(&["--lib", "--bins", "--examples", "--tests", "--benches"]);
+ }
+ DocTests::Yes => {}
+ }
+
+ cmd.arg("--").args(&builder.config.cmd.test_args());
+ // rustbuild tests are racy on directory creation so just run them one at a time.
+ // Since there's not many this shouldn't be a problem.
+ cmd.arg("--test-threads=1");
+ try_run(builder, &mut cmd);
+ }
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/bootstrap")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Bootstrap);
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct TierCheck {
+ pub compiler: Compiler,
+}
+
+impl Step for TierCheck {
+ type Output = ();
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/tools/tier-check")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ let compiler =
+ run.builder.compiler_for(run.builder.top_stage, run.builder.build.build, run.target);
+ run.builder.ensure(TierCheck { compiler });
+ }
+
+ /// Tests the Platform Support page in the rustc book.
+ fn run(self, builder: &Builder<'_>) {
+ builder.ensure(compile::Std::new(self.compiler, self.compiler.host));
+ let mut cargo = tool::prepare_tool_cargo(
+ builder,
+ self.compiler,
+ Mode::ToolStd,
+ self.compiler.host,
+ "run",
+ "src/tools/tier-check",
+ SourceType::InTree,
+ &[],
+ );
+ cargo.arg(builder.src.join("src/doc/rustc/src/platform-support.md"));
+ cargo.arg(&builder.rustc(self.compiler));
+ if builder.is_verbose() {
+ cargo.arg("--verbose");
+ }
+
+ builder.info("platform support check");
+ try_run(builder, &mut cargo.into());
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct LintDocs {
+ pub compiler: Compiler,
+ pub target: TargetSelection,
+}
+
+impl Step for LintDocs {
+ type Output = ();
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/tools/lint-docs")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(LintDocs {
+ compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
+ target: run.target,
+ });
+ }
+
+ /// Tests that the lint examples in the rustc book generate the correct
+ /// lints and have the expected format.
+ fn run(self, builder: &Builder<'_>) {
+ builder.ensure(crate::doc::RustcBook {
+ compiler: self.compiler,
+ target: self.target,
+ validate: true,
+ });
+ }
+}
diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs
new file mode 100644
index 000000000..06fa5039f
--- /dev/null
+++ b/src/bootstrap/tool.rs
@@ -0,0 +1,918 @@
+use std::collections::HashSet;
+use std::env;
+use std::fs;
+use std::path::PathBuf;
+use std::process::Command;
+
+use crate::builder::{Builder, Cargo as CargoCommand, RunConfig, ShouldRun, Step};
+use crate::channel::GitInfo;
+use crate::compile;
+use crate::config::TargetSelection;
+use crate::toolstate::ToolState;
+use crate::util::{add_dylib_path, exe, t};
+use crate::Compiler;
+use crate::Mode;
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub enum SourceType {
+ InTree,
+ Submodule,
+}
+
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
+struct ToolBuild {
+ compiler: Compiler,
+ target: TargetSelection,
+ tool: &'static str,
+ path: &'static str,
+ mode: Mode,
+ is_optional_tool: bool,
+ source_type: SourceType,
+ extra_features: Vec<String>,
+}
+
+impl Step for ToolBuild {
+ type Output = Option<PathBuf>;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.never()
+ }
+
+ /// Builds a tool in `src/tools`
+ ///
+ /// This will build the specified tool with the specified `host` compiler in
+ /// `stage` into the normal cargo output directory.
+ fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
+ let compiler = self.compiler;
+ let target = self.target;
+ let mut tool = self.tool;
+ let path = self.path;
+ let is_optional_tool = self.is_optional_tool;
+
+ match self.mode {
+ Mode::ToolRustc => {
+ builder.ensure(compile::Std::new(compiler, compiler.host));
+ builder.ensure(compile::Rustc::new(compiler, target));
+ }
+ Mode::ToolStd => builder.ensure(compile::Std::new(compiler, target)),
+ Mode::ToolBootstrap => {} // uses downloaded stage0 compiler libs
+ _ => panic!("unexpected Mode for tool build"),
+ }
+
+ let cargo = prepare_tool_cargo(
+ builder,
+ compiler,
+ self.mode,
+ target,
+ "build",
+ path,
+ self.source_type,
+ &self.extra_features,
+ );
+
+ builder.info(&format!("Building stage{} tool {} ({})", compiler.stage, tool, target));
+ let mut duplicates = Vec::new();
+ let is_expected = compile::stream_cargo(builder, cargo, vec![], &mut |msg| {
+ // Only care about big things like the RLS/Cargo for now
+ match tool {
+ "rls" | "cargo" | "clippy-driver" | "miri" | "rustfmt" => {}
+
+ _ => return,
+ }
+ let (id, features, filenames) = match msg {
+ compile::CargoMessage::CompilerArtifact {
+ package_id,
+ features,
+ filenames,
+ target: _,
+ } => (package_id, features, filenames),
+ _ => return,
+ };
+ let features = features.iter().map(|s| s.to_string()).collect::<Vec<_>>();
+
+ for path in filenames {
+ let val = (tool, PathBuf::from(&*path), features.clone());
+ // we're only interested in deduplicating rlibs for now
+ if val.1.extension().and_then(|s| s.to_str()) != Some("rlib") {
+ continue;
+ }
+
+ // Don't worry about compiles that turn out to be host
+ // dependencies or build scripts. To skip these we look for
+ // anything that goes in `.../release/deps` but *doesn't* go in
+ // `$target/release/deps`. This ensure that outputs in
+ // `$target/release` are still considered candidates for
+ // deduplication.
+ if let Some(parent) = val.1.parent() {
+ if parent.ends_with("release/deps") {
+ let maybe_target = parent
+ .parent()
+ .and_then(|p| p.parent())
+ .and_then(|p| p.file_name())
+ .and_then(|p| p.to_str())
+ .unwrap();
+ if maybe_target != &*target.triple {
+ continue;
+ }
+ }
+ }
+
+ // Record that we've built an artifact for `id`, and if one was
+ // already listed then we need to see if we reused the same
+ // artifact or produced a duplicate.
+ let mut artifacts = builder.tool_artifacts.borrow_mut();
+ let prev_artifacts = artifacts.entry(target).or_default();
+ let prev = match prev_artifacts.get(&*id) {
+ Some(prev) => prev,
+ None => {
+ prev_artifacts.insert(id.to_string(), val);
+ continue;
+ }
+ };
+ if prev.1 == val.1 {
+ return; // same path, same artifact
+ }
+
+ // If the paths are different and one of them *isn't* inside of
+ // `release/deps`, then it means it's probably in
+ // `$target/release`, or it's some final artifact like
+ // `libcargo.rlib`. In these situations Cargo probably just
+ // copied it up from `$target/release/deps/libcargo-xxxx.rlib`,
+ // so if the features are equal we can just skip it.
+ let prev_no_hash = prev.1.parent().unwrap().ends_with("release/deps");
+ let val_no_hash = val.1.parent().unwrap().ends_with("release/deps");
+ if prev.2 == val.2 || !prev_no_hash || !val_no_hash {
+ return;
+ }
+
+ // ... and otherwise this looks like we duplicated some sort of
+ // compilation, so record it to generate an error later.
+ duplicates.push((id.to_string(), val, prev.clone()));
+ }
+ });
+
+ if is_expected && !duplicates.is_empty() {
+ eprintln!(
+ "duplicate artifacts found when compiling a tool, this \
+ typically means that something was recompiled because \
+ a transitive dependency has different features activated \
+ than in a previous build:\n"
+ );
+ let (same, different): (Vec<_>, Vec<_>) =
+ duplicates.into_iter().partition(|(_, cur, prev)| cur.2 == prev.2);
+ if !same.is_empty() {
+ eprintln!(
+ "the following dependencies are duplicated although they \
+ have the same features enabled:"
+ );
+ for (id, cur, prev) in same {
+ eprintln!(" {}", id);
+ // same features
+ eprintln!(" `{}` ({:?})\n `{}` ({:?})", cur.0, cur.1, prev.0, prev.1);
+ }
+ }
+ if !different.is_empty() {
+ eprintln!("the following dependencies have different features:");
+ for (id, cur, prev) in different {
+ eprintln!(" {}", id);
+ let cur_features: HashSet<_> = cur.2.into_iter().collect();
+ let prev_features: HashSet<_> = prev.2.into_iter().collect();
+ eprintln!(
+ " `{}` additionally enabled features {:?} at {:?}",
+ cur.0,
+ &cur_features - &prev_features,
+ cur.1
+ );
+ eprintln!(
+ " `{}` additionally enabled features {:?} at {:?}",
+ prev.0,
+ &prev_features - &cur_features,
+ prev.1
+ );
+ }
+ }
+ eprintln!();
+ eprintln!(
+ "to fix this you will probably want to edit the local \
+ src/tools/rustc-workspace-hack/Cargo.toml crate, as \
+ that will update the dependency graph to ensure that \
+ these crates all share the same feature set"
+ );
+ panic!("tools should not compile multiple copies of the same crate");
+ }
+
+ builder.save_toolstate(
+ tool,
+ if is_expected { ToolState::TestFail } else { ToolState::BuildFail },
+ );
+
+ if !is_expected {
+ if !is_optional_tool {
+ crate::detail_exit(1);
+ } else {
+ None
+ }
+ } else {
+ // HACK(#82501): on Windows, the tools directory gets added to PATH when running tests, and
+ // compiletest confuses HTML tidy with the in-tree tidy. Name the in-tree tidy something
+ // different so the problem doesn't come up.
+ if tool == "tidy" {
+ tool = "rust-tidy";
+ }
+ let cargo_out = builder.cargo_out(compiler, self.mode, target).join(exe(tool, target));
+ let bin = builder.tools_dir(compiler).join(exe(tool, target));
+ builder.copy(&cargo_out, &bin);
+ Some(bin)
+ }
+ }
+}
+
+pub fn prepare_tool_cargo(
+ builder: &Builder<'_>,
+ compiler: Compiler,
+ mode: Mode,
+ target: TargetSelection,
+ command: &'static str,
+ path: &'static str,
+ source_type: SourceType,
+ extra_features: &[String],
+) -> CargoCommand {
+ let mut cargo = builder.cargo(compiler, mode, source_type, target, command);
+ let dir = builder.src.join(path);
+ cargo.arg("--manifest-path").arg(dir.join("Cargo.toml"));
+
+ let mut features = extra_features.to_vec();
+ if builder.build.config.cargo_native_static {
+ if path.ends_with("cargo")
+ || path.ends_with("rls")
+ || path.ends_with("clippy")
+ || path.ends_with("miri")
+ || path.ends_with("rustfmt")
+ {
+ cargo.env("LIBZ_SYS_STATIC", "1");
+ features.push("rustc-workspace-hack/all-static".to_string());
+ }
+ }
+
+ // clippy tests need to know about the stage sysroot. Set them consistently while building to
+ // avoid rebuilding when running tests.
+ cargo.env("SYSROOT", builder.sysroot(compiler));
+
+ // if tools are using lzma we want to force the build script to build its
+ // own copy
+ cargo.env("LZMA_API_STATIC", "1");
+
+ // CFG_RELEASE is needed by rustfmt (and possibly other tools) which
+ // import rustc-ap-rustc_attr which requires this to be set for the
+ // `#[cfg(version(...))]` attribute.
+ cargo.env("CFG_RELEASE", builder.rust_release());
+ cargo.env("CFG_RELEASE_CHANNEL", &builder.config.channel);
+ cargo.env("CFG_VERSION", builder.rust_version());
+ cargo.env("CFG_RELEASE_NUM", &builder.version);
+ cargo.env("DOC_RUST_LANG_ORG_CHANNEL", builder.doc_rust_lang_org_channel());
+
+ let info = GitInfo::new(builder.config.ignore_git, &dir);
+ if let Some(sha) = info.sha() {
+ cargo.env("CFG_COMMIT_HASH", sha);
+ }
+ if let Some(sha_short) = info.sha_short() {
+ cargo.env("CFG_SHORT_COMMIT_HASH", sha_short);
+ }
+ if let Some(date) = info.commit_date() {
+ cargo.env("CFG_COMMIT_DATE", date);
+ }
+ if !features.is_empty() {
+ cargo.arg("--features").arg(&features.join(", "));
+ }
+ cargo
+}
+
+macro_rules! bootstrap_tool {
+ ($(
+ $name:ident, $path:expr, $tool_name:expr
+ $(,is_external_tool = $external:expr)*
+ $(,is_unstable_tool = $unstable:expr)*
+ ;
+ )+) => {
+ #[derive(Copy, PartialEq, Eq, Clone)]
+ pub enum Tool {
+ $(
+ $name,
+ )+
+ }
+
+ impl<'a> Builder<'a> {
+ pub fn tool_exe(&self, tool: Tool) -> PathBuf {
+ match tool {
+ $(Tool::$name =>
+ self.ensure($name {
+ compiler: self.compiler(0, self.config.build),
+ target: self.config.build,
+ }),
+ )+
+ }
+ }
+ }
+
+ $(
+ #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+ pub struct $name {
+ pub compiler: Compiler,
+ pub target: TargetSelection,
+ }
+
+ impl Step for $name {
+ type Output = PathBuf;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path($path)
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure($name {
+ // snapshot compiler
+ compiler: run.builder.compiler(0, run.builder.config.build),
+ target: run.target,
+ });
+ }
+
+ fn run(self, builder: &Builder<'_>) -> PathBuf {
+ builder.ensure(ToolBuild {
+ compiler: self.compiler,
+ target: self.target,
+ tool: $tool_name,
+ mode: if false $(|| $unstable)* {
+ // use in-tree libraries for unstable features
+ Mode::ToolStd
+ } else {
+ Mode::ToolBootstrap
+ },
+ path: $path,
+ is_optional_tool: false,
+ source_type: if false $(|| $external)* {
+ SourceType::Submodule
+ } else {
+ SourceType::InTree
+ },
+ extra_features: vec![],
+ }).expect("expected to build -- essential tool")
+ }
+ }
+ )+
+ }
+}
+
+bootstrap_tool!(
+ Rustbook, "src/tools/rustbook", "rustbook";
+ UnstableBookGen, "src/tools/unstable-book-gen", "unstable-book-gen";
+ Tidy, "src/tools/tidy", "tidy";
+ Linkchecker, "src/tools/linkchecker", "linkchecker";
+ CargoTest, "src/tools/cargotest", "cargotest";
+ Compiletest, "src/tools/compiletest", "compiletest", is_unstable_tool = true;
+ BuildManifest, "src/tools/build-manifest", "build-manifest";
+ RemoteTestClient, "src/tools/remote-test-client", "remote-test-client";
+ RustInstaller, "src/tools/rust-installer", "rust-installer", is_external_tool = true;
+ RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes";
+ ExpandYamlAnchors, "src/tools/expand-yaml-anchors", "expand-yaml-anchors";
+ LintDocs, "src/tools/lint-docs", "lint-docs";
+ JsonDocCk, "src/tools/jsondocck", "jsondocck";
+ HtmlChecker, "src/tools/html-checker", "html-checker";
+ BumpStage0, "src/tools/bump-stage0", "bump-stage0";
+);
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)]
+pub struct ErrorIndex {
+ pub compiler: Compiler,
+}
+
+impl ErrorIndex {
+ pub fn command(builder: &Builder<'_>) -> Command {
+ // Error-index-generator links with the rustdoc library, so we need to add `rustc_lib_paths`
+ // for rustc_private and libLLVM.so, and `sysroot_lib` for libstd, etc.
+ let host = builder.config.build;
+ let compiler = builder.compiler_for(builder.top_stage, host, host);
+ let mut cmd = Command::new(builder.ensure(ErrorIndex { compiler }));
+ let mut dylib_paths = builder.rustc_lib_paths(compiler);
+ dylib_paths.push(PathBuf::from(&builder.sysroot_libdir(compiler, compiler.host)));
+ add_dylib_path(dylib_paths, &mut cmd);
+ cmd
+ }
+}
+
+impl Step for ErrorIndex {
+ type Output = PathBuf;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/tools/error_index_generator")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ // Compile the error-index in the same stage as rustdoc to avoid
+ // recompiling rustdoc twice if we can.
+ //
+ // NOTE: This `make_run` isn't used in normal situations, only if you
+ // manually build the tool with `x.py build
+ // src/tools/error-index-generator` which almost nobody does.
+ // Normally, `x.py test` or `x.py doc` will use the
+ // `ErrorIndex::command` function instead.
+ let compiler =
+ run.builder.compiler(run.builder.top_stage.saturating_sub(1), run.builder.config.build);
+ run.builder.ensure(ErrorIndex { compiler });
+ }
+
+ fn run(self, builder: &Builder<'_>) -> PathBuf {
+ builder
+ .ensure(ToolBuild {
+ compiler: self.compiler,
+ target: self.compiler.host,
+ tool: "error_index_generator",
+ mode: Mode::ToolRustc,
+ path: "src/tools/error_index_generator",
+ is_optional_tool: false,
+ source_type: SourceType::InTree,
+ extra_features: Vec::new(),
+ })
+ .expect("expected to build -- essential tool")
+ }
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RemoteTestServer {
+ pub compiler: Compiler,
+ pub target: TargetSelection,
+}
+
+impl Step for RemoteTestServer {
+ type Output = PathBuf;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/tools/remote-test-server")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(RemoteTestServer {
+ compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
+ target: run.target,
+ });
+ }
+
+ fn run(self, builder: &Builder<'_>) -> PathBuf {
+ builder
+ .ensure(ToolBuild {
+ compiler: self.compiler,
+ target: self.target,
+ tool: "remote-test-server",
+ mode: Mode::ToolStd,
+ path: "src/tools/remote-test-server",
+ is_optional_tool: false,
+ source_type: SourceType::InTree,
+ extra_features: Vec::new(),
+ })
+ .expect("expected to build -- essential tool")
+ }
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)]
+pub struct Rustdoc {
+ /// This should only ever be 0 or 2.
+ /// We sometimes want to reference the "bootstrap" rustdoc, which is why this option is here.
+ pub compiler: Compiler,
+}
+
+impl Step for Rustdoc {
+ type Output = PathBuf;
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/tools/rustdoc").path("src/librustdoc")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Rustdoc {
+ // Note: this is somewhat unique in that we actually want a *target*
+ // compiler here, because rustdoc *is* a compiler. We won't be using
+ // this as the compiler to build with, but rather this is "what
+ // compiler are we producing"?
+ compiler: run.builder.compiler(run.builder.top_stage, run.target),
+ });
+ }
+
+ fn run(self, builder: &Builder<'_>) -> PathBuf {
+ let target_compiler = self.compiler;
+ if target_compiler.stage == 0 {
+ if !target_compiler.is_snapshot(builder) {
+ panic!("rustdoc in stage 0 must be snapshot rustdoc");
+ }
+ return builder.initial_rustc.with_file_name(exe("rustdoc", target_compiler.host));
+ }
+ let target = target_compiler.host;
+ // Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise
+ // we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage
+ // compilers, which isn't what we want. Rustdoc should be linked in the same way as the
+ // rustc compiler it's paired with, so it must be built with the previous stage compiler.
+ let build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build);
+
+ // When using `download-rustc` and a stage0 build_compiler, copying rustc doesn't actually
+ // build stage0 libstd (because the libstd in sysroot has the wrong ABI). Explicitly build
+ // it.
+ builder.ensure(compile::Std::new(build_compiler, target_compiler.host));
+ builder.ensure(compile::Rustc::new(build_compiler, target_compiler.host));
+ // NOTE: this implies that `download-rustc` is pretty useless when compiling with the stage0
+ // compiler, since you do just as much work.
+ if !builder.config.dry_run && builder.download_rustc() && build_compiler.stage == 0 {
+ println!(
+ "warning: `download-rustc` does nothing when building stage1 tools; consider using `--stage 2` instead"
+ );
+ }
+
+ // The presence of `target_compiler` ensures that the necessary libraries (codegen backends,
+ // compiler libraries, ...) are built. Rustdoc does not require the presence of any
+ // libraries within sysroot_libdir (i.e., rustlib), though doctests may want it (since
+ // they'll be linked to those libraries). As such, don't explicitly `ensure` any additional
+ // libraries here. The intuition here is that If we've built a compiler, we should be able
+ // to build rustdoc.
+ //
+ let mut features = Vec::new();
+ if builder.config.jemalloc {
+ features.push("jemalloc".to_string());
+ }
+
+ let cargo = prepare_tool_cargo(
+ builder,
+ build_compiler,
+ Mode::ToolRustc,
+ target,
+ "build",
+ "src/tools/rustdoc",
+ SourceType::InTree,
+ features.as_slice(),
+ );
+
+ builder.info(&format!(
+ "Building rustdoc for stage{} ({})",
+ target_compiler.stage, target_compiler.host
+ ));
+ builder.run(&mut cargo.into());
+
+ // Cargo adds a number of paths to the dylib search path on windows, which results in
+ // the wrong rustdoc being executed. To avoid the conflicting rustdocs, we name the "tool"
+ // rustdoc a different name.
+ let tool_rustdoc = builder
+ .cargo_out(build_compiler, Mode::ToolRustc, target)
+ .join(exe("rustdoc_tool_binary", target_compiler.host));
+
+ // don't create a stage0-sysroot/bin directory.
+ if target_compiler.stage > 0 {
+ let sysroot = builder.sysroot(target_compiler);
+ let bindir = sysroot.join("bin");
+ t!(fs::create_dir_all(&bindir));
+ let bin_rustdoc = bindir.join(exe("rustdoc", target_compiler.host));
+ let _ = fs::remove_file(&bin_rustdoc);
+ builder.copy(&tool_rustdoc, &bin_rustdoc);
+ bin_rustdoc
+ } else {
+ tool_rustdoc
+ }
+ }
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Cargo {
+ pub compiler: Compiler,
+ pub target: TargetSelection,
+}
+
+impl Step for Cargo {
+ type Output = PathBuf;
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let builder = run.builder;
+ run.path("src/tools/cargo").default_condition(
+ builder.config.extended
+ && builder.config.tools.as_ref().map_or(
+ true,
+ // If `tools` is set, search list for this tool.
+ |tools| tools.iter().any(|tool| tool == "cargo"),
+ ),
+ )
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(Cargo {
+ compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
+ target: run.target,
+ });
+ }
+
+ fn run(self, builder: &Builder<'_>) -> PathBuf {
+ let cargo_bin_path = builder
+ .ensure(ToolBuild {
+ compiler: self.compiler,
+ target: self.target,
+ tool: "cargo",
+ mode: Mode::ToolRustc,
+ path: "src/tools/cargo",
+ is_optional_tool: false,
+ source_type: SourceType::Submodule,
+ extra_features: Vec::new(),
+ })
+ .expect("expected to build -- essential tool");
+
+ let build_cred = |name, path| {
+ // These credential helpers are currently experimental.
+ // Any build failures will be ignored.
+ let _ = builder.ensure(ToolBuild {
+ compiler: self.compiler,
+ target: self.target,
+ tool: name,
+ mode: Mode::ToolRustc,
+ path,
+ is_optional_tool: true,
+ source_type: SourceType::Submodule,
+ extra_features: Vec::new(),
+ });
+ };
+
+ if self.target.contains("windows") {
+ build_cred(
+ "cargo-credential-wincred",
+ "src/tools/cargo/crates/credential/cargo-credential-wincred",
+ );
+ }
+ if self.target.contains("apple-darwin") {
+ build_cred(
+ "cargo-credential-macos-keychain",
+ "src/tools/cargo/crates/credential/cargo-credential-macos-keychain",
+ );
+ }
+ build_cred(
+ "cargo-credential-1password",
+ "src/tools/cargo/crates/credential/cargo-credential-1password",
+ );
+ cargo_bin_path
+ }
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct LldWrapper {
+ pub compiler: Compiler,
+ pub target: TargetSelection,
+}
+
+impl Step for LldWrapper {
+ type Output = PathBuf;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.never()
+ }
+
+ fn run(self, builder: &Builder<'_>) -> PathBuf {
+ let src_exe = builder
+ .ensure(ToolBuild {
+ compiler: self.compiler,
+ target: self.target,
+ tool: "lld-wrapper",
+ mode: Mode::ToolStd,
+ path: "src/tools/lld-wrapper",
+ is_optional_tool: false,
+ source_type: SourceType::InTree,
+ extra_features: Vec::new(),
+ })
+ .expect("expected to build -- essential tool");
+
+ src_exe
+ }
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RustAnalyzer {
+ pub compiler: Compiler,
+ pub target: TargetSelection,
+}
+
+impl Step for RustAnalyzer {
+ type Output = Option<PathBuf>;
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = false;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let builder = run.builder;
+ run.path("src/tools/rust-analyzer").default_condition(
+ builder.config.extended
+ && builder
+ .config
+ .tools
+ .as_ref()
+ .map_or(true, |tools| tools.iter().any(|tool| tool == "rust-analyzer")),
+ )
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(RustAnalyzer {
+ compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
+ target: run.target,
+ });
+ }
+
+ fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
+ builder.ensure(ToolBuild {
+ compiler: self.compiler,
+ target: self.target,
+ tool: "rust-analyzer",
+ mode: Mode::ToolStd,
+ path: "src/tools/rust-analyzer",
+ extra_features: vec!["rust-analyzer/in-rust-tree".to_owned()],
+ is_optional_tool: false,
+ source_type: SourceType::InTree,
+ })
+ }
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RustAnalyzerProcMacroSrv {
+ pub compiler: Compiler,
+ pub target: TargetSelection,
+}
+
+impl Step for RustAnalyzerProcMacroSrv {
+ type Output = Option<PathBuf>;
+ const DEFAULT: bool = true;
+ const ONLY_HOSTS: bool = false;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let builder = run.builder;
+ run.path("src/tools/rust-analyzer").default_condition(
+ builder.config.extended
+ && builder
+ .config
+ .tools
+ .as_ref()
+ .map_or(true, |tools| tools.iter().any(|tool| tool == "rust-analyzer")),
+ )
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(RustAnalyzerProcMacroSrv {
+ compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
+ target: run.target,
+ });
+ }
+
+ fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
+ builder.ensure(ToolBuild {
+ compiler: self.compiler,
+ target: self.target,
+ tool: "rust-analyzer-proc-macro-srv",
+ mode: Mode::ToolStd,
+ path: "src/tools/rust-analyzer/crates/proc-macro-srv-cli",
+ extra_features: vec!["proc-macro-srv/sysroot-abi".to_owned()],
+ is_optional_tool: false,
+ source_type: SourceType::InTree,
+ })
+ }
+}
+
+macro_rules! tool_extended {
+ (($sel:ident, $builder:ident),
+ $($name:ident,
+ $toolstate:ident,
+ $path:expr,
+ $tool_name:expr,
+ stable = $stable:expr,
+ $(in_tree = $in_tree:expr,)?
+ $(submodule = $submodule:literal,)?
+ $(tool_std = $tool_std:literal,)?
+ $extra_deps:block;)+) => {
+ $(
+ #[derive(Debug, Clone, Hash, PartialEq, Eq)]
+ pub struct $name {
+ pub compiler: Compiler,
+ pub target: TargetSelection,
+ pub extra_features: Vec<String>,
+ }
+
+ impl Step for $name {
+ type Output = Option<PathBuf>;
+ const DEFAULT: bool = true; // Overwritten below
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ let builder = run.builder;
+ run.path($path).default_condition(
+ builder.config.extended
+ && builder.config.tools.as_ref().map_or(
+ // By default, on nightly/dev enable all tools, else only
+ // build stable tools.
+ $stable || builder.build.unstable_features(),
+ // If `tools` is set, search list for this tool.
+ |tools| {
+ tools.iter().any(|tool| match tool.as_ref() {
+ "clippy" => $tool_name == "clippy-driver",
+ x => $tool_name == x,
+ })
+ }),
+ )
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure($name {
+ compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
+ target: run.target,
+ extra_features: Vec::new(),
+ });
+ }
+
+ #[allow(unused_mut)]
+ fn run(mut $sel, $builder: &Builder<'_>) -> Option<PathBuf> {
+ $extra_deps
+ $( $builder.update_submodule(&Path::new("src").join("tools").join($submodule)); )?
+ $builder.ensure(ToolBuild {
+ compiler: $sel.compiler,
+ target: $sel.target,
+ tool: $tool_name,
+ mode: if false $(|| $tool_std)? { Mode::ToolStd } else { Mode::ToolRustc },
+ path: $path,
+ extra_features: $sel.extra_features,
+ is_optional_tool: true,
+ source_type: if false $(|| $in_tree)* {
+ SourceType::InTree
+ } else {
+ SourceType::Submodule
+ },
+ })
+ }
+ }
+ )+
+ }
+}
+
+// Note: tools need to be also added to `Builder::get_step_descriptions` in `builder.rs`
+// to make `./x.py build <tool>` work.
+// Note: Most submodule updates for tools are handled by bootstrap.py, since they're needed just to
+// invoke Cargo to build bootstrap. See the comment there for more details.
+tool_extended!((self, builder),
+ Cargofmt, rustfmt, "src/tools/rustfmt", "cargo-fmt", stable=true, in_tree=true, {};
+ CargoClippy, clippy, "src/tools/clippy", "cargo-clippy", stable=true, in_tree=true, {};
+ Clippy, clippy, "src/tools/clippy", "clippy-driver", stable=true, in_tree=true, {};
+ Miri, miri, "src/tools/miri", "miri", stable=false, {};
+ CargoMiri, miri, "src/tools/miri/cargo-miri", "cargo-miri", stable=false, {};
+ Rls, rls, "src/tools/rls", "rls", stable=true, {
+ builder.ensure(Clippy {
+ compiler: self.compiler,
+ target: self.target,
+ extra_features: Vec::new(),
+ });
+ self.extra_features.push("clippy".to_owned());
+ };
+ // FIXME: tool_std is not quite right, we shouldn't allow nightly features.
+ // But `builder.cargo` doesn't know how to handle ToolBootstrap in stages other than 0,
+ // and this is close enough for now.
+ RustDemangler, rust_demangler, "src/tools/rust-demangler", "rust-demangler", stable=false, in_tree=true, tool_std=true, {};
+ Rustfmt, rustfmt, "src/tools/rustfmt", "rustfmt", stable=true, in_tree=true, {};
+);
+
+impl<'a> Builder<'a> {
+ /// Gets a `Command` which is ready to run `tool` in `stage` built for
+ /// `host`.
+ pub fn tool_cmd(&self, tool: Tool) -> Command {
+ let mut cmd = Command::new(self.tool_exe(tool));
+ let compiler = self.compiler(0, self.config.build);
+ let host = &compiler.host;
+ // Prepares the `cmd` provided to be able to run the `compiler` provided.
+ //
+ // Notably this munges the dynamic library lookup path to point to the
+ // right location to run `compiler`.
+ let mut lib_paths: Vec<PathBuf> = vec![
+ self.build.rustc_snapshot_libdir(),
+ self.cargo_out(compiler, Mode::ToolBootstrap, *host).join("deps"),
+ ];
+
+ // On MSVC a tool may invoke a C compiler (e.g., compiletest in run-make
+ // mode) and that C compiler may need some extra PATH modification. Do
+ // so here.
+ if compiler.host.contains("msvc") {
+ let curpaths = env::var_os("PATH").unwrap_or_default();
+ let curpaths = env::split_paths(&curpaths).collect::<Vec<_>>();
+ for &(ref k, ref v) in self.cc[&compiler.host].env() {
+ if k != "PATH" {
+ continue;
+ }
+ for path in env::split_paths(v) {
+ if !curpaths.contains(&path) {
+ lib_paths.push(path);
+ }
+ }
+ }
+ }
+
+ add_dylib_path(lib_paths, &mut cmd);
+
+ // Provide a RUSTC for this command to use.
+ cmd.env("RUSTC", &self.initial_rustc);
+
+ cmd
+ }
+}
diff --git a/src/bootstrap/toolstate.rs b/src/bootstrap/toolstate.rs
new file mode 100644
index 000000000..2cfeae7dc
--- /dev/null
+++ b/src/bootstrap/toolstate.rs
@@ -0,0 +1,479 @@
+use crate::builder::{Builder, RunConfig, ShouldRun, Step};
+use crate::util::t;
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::env;
+use std::fmt;
+use std::fs;
+use std::io::{Seek, SeekFrom};
+use std::path::{Path, PathBuf};
+use std::process::Command;
+use std::time;
+
+// Each cycle is 42 days long (6 weeks); the last week is 35..=42 then.
+const BETA_WEEK_START: u64 = 35;
+
+#[cfg(target_os = "linux")]
+const OS: Option<&str> = Some("linux");
+
+#[cfg(windows)]
+const OS: Option<&str> = Some("windows");
+
+#[cfg(all(not(target_os = "linux"), not(windows)))]
+const OS: Option<&str> = None;
+
+type ToolstateData = HashMap<Box<str>, ToolState>;
+
+#[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, PartialOrd)]
+#[serde(rename_all = "kebab-case")]
+/// Whether a tool can be compiled, tested or neither
+pub enum ToolState {
+ /// The tool compiles successfully, but the test suite fails
+ TestFail = 1,
+ /// The tool compiles successfully and its test suite passes
+ TestPass = 2,
+ /// The tool can't even be compiled
+ BuildFail = 0,
+}
+
+impl fmt::Display for ToolState {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ write!(
+ f,
+ "{}",
+ match self {
+ ToolState::TestFail => "test-fail",
+ ToolState::TestPass => "test-pass",
+ ToolState::BuildFail => "build-fail",
+ }
+ )
+ }
+}
+
+/// Number of days after the last promotion of beta.
+/// Its value is 41 on the Tuesday where "Promote master to beta (T-2)" happens.
+/// The Wednesday after this has value 0.
+/// We track this value to prevent regressing tools in the last week of the 6-week cycle.
+fn days_since_beta_promotion() -> u64 {
+ let since_epoch = t!(time::SystemTime::UNIX_EPOCH.elapsed());
+ (since_epoch.as_secs() / 86400 - 20) % 42
+}
+
+// These tools must test-pass on the beta/stable channels.
+//
+// On the nightly channel, their build step must be attempted, but they may not
+// be able to build successfully.
+static STABLE_TOOLS: &[(&str, &str)] = &[
+ ("book", "src/doc/book"),
+ ("nomicon", "src/doc/nomicon"),
+ ("reference", "src/doc/reference"),
+ ("rust-by-example", "src/doc/rust-by-example"),
+ ("edition-guide", "src/doc/edition-guide"),
+ ("rls", "src/tools/rls"),
+];
+
+// These tools are permitted to not build on the beta/stable channels.
+//
+// We do require that we checked whether they build or not on the tools builder,
+// though, as otherwise we will be unable to file an issue if they start
+// failing.
+static NIGHTLY_TOOLS: &[(&str, &str)] = &[
+ ("miri", "src/tools/miri"),
+ ("embedded-book", "src/doc/embedded-book"),
+ // ("rustc-dev-guide", "src/doc/rustc-dev-guide"),
+];
+
+fn print_error(tool: &str, submodule: &str) {
+ eprintln!();
+ eprintln!("We detected that this PR updated '{}', but its tests failed.", tool);
+ eprintln!();
+ eprintln!("If you do intend to update '{}', please check the error messages above and", tool);
+ eprintln!("commit another update.");
+ eprintln!();
+ eprintln!("If you do NOT intend to update '{}', please ensure you did not accidentally", tool);
+ eprintln!("change the submodule at '{}'. You may ask your reviewer for the", submodule);
+ eprintln!("proper steps.");
+ crate::detail_exit(3);
+}
+
+fn check_changed_files(toolstates: &HashMap<Box<str>, ToolState>) {
+ // Changed files
+ let output = std::process::Command::new("git")
+ .arg("diff")
+ .arg("--name-status")
+ .arg("HEAD")
+ .arg("HEAD^")
+ .output();
+ let output = match output {
+ Ok(o) => o,
+ Err(e) => {
+ eprintln!("Failed to get changed files: {:?}", e);
+ crate::detail_exit(1);
+ }
+ };
+
+ let output = t!(String::from_utf8(output.stdout));
+
+ for (tool, submodule) in STABLE_TOOLS.iter().chain(NIGHTLY_TOOLS.iter()) {
+ let changed = output.lines().any(|l| l.starts_with('M') && l.ends_with(submodule));
+ eprintln!("Verifying status of {}...", tool);
+ if !changed {
+ continue;
+ }
+
+ eprintln!("This PR updated '{}', verifying if status is 'test-pass'...", submodule);
+ if toolstates[*tool] != ToolState::TestPass {
+ print_error(tool, submodule);
+ }
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct ToolStateCheck;
+
+impl Step for ToolStateCheck {
+ type Output = ();
+
+ /// Checks tool state status.
+ ///
+ /// This is intended to be used in the `checktools.sh` script. To use
+ /// this, set `save-toolstates` in `config.toml` so that tool status will
+ /// be saved to a JSON file. Then, run `x.py test --no-fail-fast` for all
+ /// of the tools to populate the JSON file. After that is done, this
+ /// command can be run to check for any status failures, and exits with an
+ /// error if there are any.
+ ///
+ /// This also handles publishing the results to the `history` directory of
+ /// the toolstate repo <https://github.com/rust-lang-nursery/rust-toolstate>
+ /// if the env var `TOOLSTATE_PUBLISH` is set. Note that there is a
+ /// *separate* step of updating the `latest.json` file and creating GitHub
+ /// issues and comments in `src/ci/publish_toolstate.sh`, which is only
+ /// performed on master. (The shell/python code is intended to be migrated
+ /// here eventually.)
+ ///
+ /// The rules for failure are:
+ /// * If the PR modifies a tool, the status must be test-pass.
+ /// NOTE: There is intent to change this, see
+ /// <https://github.com/rust-lang/rust/issues/65000>.
+ /// * All "stable" tools must be test-pass on the stable or beta branches.
+ /// * During beta promotion week, a PR is not allowed to "regress" a
+ /// stable tool. That is, the status is not allowed to get worse
+ /// (test-pass to test-fail or build-fail).
+ fn run(self, builder: &Builder<'_>) {
+ if builder.config.dry_run {
+ return;
+ }
+
+ let days_since_beta_promotion = days_since_beta_promotion();
+ let in_beta_week = days_since_beta_promotion >= BETA_WEEK_START;
+ let is_nightly = !(builder.config.channel == "beta" || builder.config.channel == "stable");
+ let toolstates = builder.toolstates();
+
+ let mut did_error = false;
+
+ for (tool, _) in STABLE_TOOLS.iter().chain(NIGHTLY_TOOLS.iter()) {
+ if !toolstates.contains_key(*tool) {
+ did_error = true;
+ eprintln!("error: Tool `{}` was not recorded in tool state.", tool);
+ }
+ }
+
+ if did_error {
+ crate::detail_exit(1);
+ }
+
+ check_changed_files(&toolstates);
+ checkout_toolstate_repo();
+ let old_toolstate = read_old_toolstate();
+
+ for (tool, _) in STABLE_TOOLS.iter() {
+ let state = toolstates[*tool];
+
+ if state != ToolState::TestPass {
+ if !is_nightly {
+ did_error = true;
+ eprintln!("error: Tool `{}` should be test-pass but is {}", tool, state);
+ } else if in_beta_week {
+ let old_state = old_toolstate
+ .iter()
+ .find(|ts| ts.tool == *tool)
+ .expect("latest.json missing tool")
+ .state();
+ if state < old_state {
+ did_error = true;
+ eprintln!(
+ "error: Tool `{}` has regressed from {} to {} during beta week.",
+ tool, old_state, state
+ );
+ } else {
+ // This warning only appears in the logs, which most
+ // people won't read. It's mostly here for testing and
+ // debugging.
+ eprintln!(
+ "warning: Tool `{}` is not test-pass (is `{}`), \
+ this should be fixed before beta is branched.",
+ tool, state
+ );
+ }
+ }
+ // `publish_toolstate.py` is responsible for updating
+ // `latest.json` and creating comments/issues warning people
+ // if there is a regression. That all happens in a separate CI
+ // job on the master branch once the PR has passed all tests
+ // on the `auto` branch.
+ }
+ }
+
+ if did_error {
+ crate::detail_exit(1);
+ }
+
+ if builder.config.channel == "nightly" && env::var_os("TOOLSTATE_PUBLISH").is_some() {
+ commit_toolstate_change(&toolstates);
+ }
+ }
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.alias("check-tools")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(ToolStateCheck);
+ }
+}
+
+impl Builder<'_> {
+ fn toolstates(&self) -> HashMap<Box<str>, ToolState> {
+ if let Some(ref path) = self.config.save_toolstates {
+ if let Some(parent) = path.parent() {
+ // Ensure the parent directory always exists
+ t!(std::fs::create_dir_all(parent));
+ }
+ let mut file =
+ t!(fs::OpenOptions::new().create(true).write(true).read(true).open(path));
+
+ serde_json::from_reader(&mut file).unwrap_or_default()
+ } else {
+ Default::default()
+ }
+ }
+
+ /// Updates the actual toolstate of a tool.
+ ///
+ /// The toolstates are saved to the file specified by the key
+ /// `rust.save-toolstates` in `config.toml`. If unspecified, nothing will be
+ /// done. The file is updated immediately after this function completes.
+ pub fn save_toolstate(&self, tool: &str, state: ToolState) {
+ // If we're in a dry run setting we don't want to save toolstates as
+ // that means if we e.g. panic down the line it'll look like we tested
+ // everything (but we actually haven't).
+ if self.config.dry_run {
+ return;
+ }
+ // Toolstate isn't tracked for clippy or rustfmt, but since most tools do, we avoid checking
+ // in all the places we could save toolstate and just do so here.
+ if tool == "clippy-driver" || tool == "rustfmt" {
+ return;
+ }
+ if let Some(ref path) = self.config.save_toolstates {
+ if let Some(parent) = path.parent() {
+ // Ensure the parent directory always exists
+ t!(std::fs::create_dir_all(parent));
+ }
+ let mut file =
+ t!(fs::OpenOptions::new().create(true).read(true).write(true).open(path));
+
+ let mut current_toolstates: HashMap<Box<str>, ToolState> =
+ serde_json::from_reader(&mut file).unwrap_or_default();
+ current_toolstates.insert(tool.into(), state);
+ t!(file.seek(SeekFrom::Start(0)));
+ t!(file.set_len(0));
+ t!(serde_json::to_writer(file, &current_toolstates));
+ }
+ }
+}
+
+fn toolstate_repo() -> String {
+ env::var("TOOLSTATE_REPO")
+ .unwrap_or_else(|_| "https://github.com/rust-lang-nursery/rust-toolstate.git".to_string())
+}
+
+/// Directory where the toolstate repo is checked out.
+const TOOLSTATE_DIR: &str = "rust-toolstate";
+
+/// Checks out the toolstate repo into `TOOLSTATE_DIR`.
+fn checkout_toolstate_repo() {
+ if let Ok(token) = env::var("TOOLSTATE_REPO_ACCESS_TOKEN") {
+ prepare_toolstate_config(&token);
+ }
+ if Path::new(TOOLSTATE_DIR).exists() {
+ eprintln!("Cleaning old toolstate directory...");
+ t!(fs::remove_dir_all(TOOLSTATE_DIR));
+ }
+
+ let status = Command::new("git")
+ .arg("clone")
+ .arg("--depth=1")
+ .arg(toolstate_repo())
+ .arg(TOOLSTATE_DIR)
+ .status();
+ let success = match status {
+ Ok(s) => s.success(),
+ Err(_) => false,
+ };
+ if !success {
+ panic!("git clone unsuccessful (status: {:?})", status);
+ }
+}
+
+/// Sets up config and authentication for modifying the toolstate repo.
+fn prepare_toolstate_config(token: &str) {
+ fn git_config(key: &str, value: &str) {
+ let status = Command::new("git").arg("config").arg("--global").arg(key).arg(value).status();
+ let success = match status {
+ Ok(s) => s.success(),
+ Err(_) => false,
+ };
+ if !success {
+ panic!("git config key={} value={} failed (status: {:?})", key, value, status);
+ }
+ }
+
+ // If changing anything here, then please check that `src/ci/publish_toolstate.sh` is up to date
+ // as well.
+ git_config("user.email", "7378925+rust-toolstate-update@users.noreply.github.com");
+ git_config("user.name", "Rust Toolstate Update");
+ git_config("credential.helper", "store");
+
+ let credential = format!("https://{}:x-oauth-basic@github.com\n", token,);
+ let git_credential_path = PathBuf::from(t!(env::var("HOME"))).join(".git-credentials");
+ t!(fs::write(&git_credential_path, credential));
+}
+
+/// Reads the latest toolstate from the toolstate repo.
+fn read_old_toolstate() -> Vec<RepoState> {
+ let latest_path = Path::new(TOOLSTATE_DIR).join("_data").join("latest.json");
+ let old_toolstate = t!(fs::read(latest_path));
+ t!(serde_json::from_slice(&old_toolstate))
+}
+
+/// This function `commit_toolstate_change` provides functionality for pushing a change
+/// to the `rust-toolstate` repository.
+///
+/// The function relies on a GitHub bot user, which should have a Personal access
+/// token defined in the environment variable $TOOLSTATE_REPO_ACCESS_TOKEN. If for
+/// some reason you need to change the token, please update the Azure Pipelines
+/// variable group.
+///
+/// 1. Generate a new Personal access token:
+///
+/// * Login to the bot account, and go to Settings -> Developer settings ->
+/// Personal access tokens
+/// * Click "Generate new token"
+/// * Enable the "public_repo" permission, then click "Generate token"
+/// * Copy the generated token (should be a 40-digit hexadecimal number).
+/// Save it somewhere secure, as the token would be gone once you leave
+/// the page.
+///
+/// 2. Update the variable group in Azure Pipelines
+///
+/// * Ping a member of the infrastructure team to do this.
+///
+/// 4. Replace the email address below if the bot account identity is changed
+///
+/// * See <https://help.github.com/articles/about-commit-email-addresses/>
+/// if a private email by GitHub is wanted.
+fn commit_toolstate_change(current_toolstate: &ToolstateData) {
+ let message = format!("({} CI update)", OS.expect("linux/windows only"));
+ let mut success = false;
+ for _ in 1..=5 {
+ // Upload the test results (the new commit-to-toolstate mapping) to the toolstate repo.
+ // This does *not* change the "current toolstate"; that only happens post-landing
+ // via `src/ci/docker/publish_toolstate.sh`.
+ publish_test_results(&current_toolstate);
+
+ // `git commit` failing means nothing to commit.
+ let status = t!(Command::new("git")
+ .current_dir(TOOLSTATE_DIR)
+ .arg("commit")
+ .arg("-a")
+ .arg("-m")
+ .arg(&message)
+ .status());
+ if !status.success() {
+ success = true;
+ break;
+ }
+
+ let status = t!(Command::new("git")
+ .current_dir(TOOLSTATE_DIR)
+ .arg("push")
+ .arg("origin")
+ .arg("master")
+ .status());
+ // If we successfully push, exit.
+ if status.success() {
+ success = true;
+ break;
+ }
+ eprintln!("Sleeping for 3 seconds before retrying push");
+ std::thread::sleep(std::time::Duration::from_secs(3));
+ let status = t!(Command::new("git")
+ .current_dir(TOOLSTATE_DIR)
+ .arg("fetch")
+ .arg("origin")
+ .arg("master")
+ .status());
+ assert!(status.success());
+ let status = t!(Command::new("git")
+ .current_dir(TOOLSTATE_DIR)
+ .arg("reset")
+ .arg("--hard")
+ .arg("origin/master")
+ .status());
+ assert!(status.success());
+ }
+
+ if !success {
+ panic!("Failed to update toolstate repository with new data");
+ }
+}
+
+/// Updates the "history" files with the latest results.
+///
+/// These results will later be promoted to `latest.json` by the
+/// `publish_toolstate.py` script if the PR passes all tests and is merged to
+/// master.
+fn publish_test_results(current_toolstate: &ToolstateData) {
+ let commit = t!(std::process::Command::new("git").arg("rev-parse").arg("HEAD").output());
+ let commit = t!(String::from_utf8(commit.stdout));
+
+ let toolstate_serialized = t!(serde_json::to_string(&current_toolstate));
+
+ let history_path = Path::new(TOOLSTATE_DIR)
+ .join("history")
+ .join(format!("{}.tsv", OS.expect("linux/windows only")));
+ let mut file = t!(fs::read_to_string(&history_path));
+ let end_of_first_line = file.find('\n').unwrap();
+ file.insert_str(end_of_first_line, &format!("\n{}\t{}", commit.trim(), toolstate_serialized));
+ t!(fs::write(&history_path, file));
+}
+
+#[derive(Debug, Deserialize)]
+struct RepoState {
+ tool: String,
+ windows: ToolState,
+ linux: ToolState,
+}
+
+impl RepoState {
+ fn state(&self) -> ToolState {
+ if cfg!(target_os = "linux") {
+ self.linux
+ } else if cfg!(windows) {
+ self.windows
+ } else {
+ unimplemented!()
+ }
+ }
+}
diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs
new file mode 100644
index 000000000..1895e2901
--- /dev/null
+++ b/src/bootstrap/util.rs
@@ -0,0 +1,603 @@
+//! Various utility functions used throughout rustbuild.
+//!
+//! Simple things like testing the various filesystem operations here and there,
+//! not a lot of interesting happenings here unfortunately.
+
+use std::env;
+use std::fs;
+use std::io;
+use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
+use std::str;
+use std::time::{Instant, SystemTime, UNIX_EPOCH};
+
+use crate::builder::Builder;
+use crate::config::{Config, TargetSelection};
+
+/// A helper macro to `unwrap` a result except also print out details like:
+///
+/// * The file/line of the panic
+/// * The expression that failed
+/// * The error itself
+///
+/// This is currently used judiciously throughout the build system rather than
+/// using a `Result` with `try!`, but this may change one day...
+#[macro_export]
+macro_rules! t {
+ ($e:expr) => {
+ match $e {
+ Ok(e) => e,
+ Err(e) => panic!("{} failed with {}", stringify!($e), e),
+ }
+ };
+ // it can show extra info in the second parameter
+ ($e:expr, $extra:expr) => {
+ match $e {
+ Ok(e) => e,
+ Err(e) => panic!("{} failed with {} ({:?})", stringify!($e), e, $extra),
+ }
+ };
+}
+pub use t;
+
+/// Given an executable called `name`, return the filename for the
+/// executable for a particular target.
+pub fn exe(name: &str, target: TargetSelection) -> String {
+ if target.contains("windows") { format!("{}.exe", name) } else { name.to_string() }
+}
+
+/// Returns `true` if the file name given looks like a dynamic library.
+pub fn is_dylib(name: &str) -> bool {
+ name.ends_with(".dylib") || name.ends_with(".so") || name.ends_with(".dll")
+}
+
+/// Returns `true` if the file name given looks like a debug info file
+pub fn is_debug_info(name: &str) -> bool {
+ // FIXME: consider split debug info on other platforms (e.g., Linux, macOS)
+ name.ends_with(".pdb")
+}
+
+/// Returns the corresponding relative library directory that the compiler's
+/// dylibs will be found in.
+pub fn libdir(target: TargetSelection) -> &'static str {
+ if target.contains("windows") { "bin" } else { "lib" }
+}
+
+/// Adds a list of lookup paths to `cmd`'s dynamic library lookup path.
+/// If the dylib_path_var is already set for this cmd, the old value will be overwritten!
+pub fn add_dylib_path(path: Vec<PathBuf>, cmd: &mut Command) {
+ let mut list = dylib_path();
+ for path in path {
+ list.insert(0, path);
+ }
+ cmd.env(dylib_path_var(), t!(env::join_paths(list)));
+}
+
+include!("dylib_util.rs");
+
+/// Adds a list of lookup paths to `cmd`'s link library lookup path.
+pub fn add_link_lib_path(path: Vec<PathBuf>, cmd: &mut Command) {
+ let mut list = link_lib_path();
+ for path in path {
+ list.insert(0, path);
+ }
+ cmd.env(link_lib_path_var(), t!(env::join_paths(list)));
+}
+
+/// Returns the environment variable which the link library lookup path
+/// resides in for this platform.
+fn link_lib_path_var() -> &'static str {
+ if cfg!(target_env = "msvc") { "LIB" } else { "LIBRARY_PATH" }
+}
+
+/// Parses the `link_lib_path_var()` environment variable, returning a list of
+/// paths that are members of this lookup path.
+fn link_lib_path() -> Vec<PathBuf> {
+ let var = match env::var_os(link_lib_path_var()) {
+ Some(v) => v,
+ None => return vec![],
+ };
+ env::split_paths(&var).collect()
+}
+
+pub struct TimeIt(bool, Instant);
+
+/// Returns an RAII structure that prints out how long it took to drop.
+pub fn timeit(builder: &Builder<'_>) -> TimeIt {
+ TimeIt(builder.config.dry_run, Instant::now())
+}
+
+impl Drop for TimeIt {
+ fn drop(&mut self) {
+ let time = self.1.elapsed();
+ if !self.0 {
+ println!("\tfinished in {}.{:03} seconds", time.as_secs(), time.subsec_millis());
+ }
+ }
+}
+
+/// Used for download caching
+pub(crate) fn program_out_of_date(stamp: &Path, key: &str) -> bool {
+ if !stamp.exists() {
+ return true;
+ }
+ t!(fs::read_to_string(stamp)) != key
+}
+
+/// Symlinks two directories, using junctions on Windows and normal symlinks on
+/// Unix.
+pub fn symlink_dir(config: &Config, src: &Path, dest: &Path) -> io::Result<()> {
+ if config.dry_run {
+ return Ok(());
+ }
+ let _ = fs::remove_dir(dest);
+ return symlink_dir_inner(src, dest);
+
+ #[cfg(not(windows))]
+ fn symlink_dir_inner(src: &Path, dest: &Path) -> io::Result<()> {
+ use std::os::unix::fs;
+ fs::symlink(src, dest)
+ }
+
+ // Creating a directory junction on windows involves dealing with reparse
+ // points and the DeviceIoControl function, and this code is a skeleton of
+ // what can be found here:
+ //
+ // http://www.flexhex.com/docs/articles/hard-links.phtml
+ #[cfg(windows)]
+ fn symlink_dir_inner(target: &Path, junction: &Path) -> io::Result<()> {
+ use std::ffi::OsStr;
+ use std::os::windows::ffi::OsStrExt;
+ use std::ptr;
+
+ use winapi::shared::minwindef::{DWORD, WORD};
+ use winapi::um::fileapi::{CreateFileW, OPEN_EXISTING};
+ use winapi::um::handleapi::CloseHandle;
+ use winapi::um::ioapiset::DeviceIoControl;
+ use winapi::um::winbase::{FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT};
+ use winapi::um::winioctl::FSCTL_SET_REPARSE_POINT;
+ use winapi::um::winnt::{
+ FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, GENERIC_WRITE,
+ IO_REPARSE_TAG_MOUNT_POINT, MAXIMUM_REPARSE_DATA_BUFFER_SIZE, WCHAR,
+ };
+
+ #[allow(non_snake_case)]
+ #[repr(C)]
+ struct REPARSE_MOUNTPOINT_DATA_BUFFER {
+ ReparseTag: DWORD,
+ ReparseDataLength: DWORD,
+ Reserved: WORD,
+ ReparseTargetLength: WORD,
+ ReparseTargetMaximumLength: WORD,
+ Reserved1: WORD,
+ ReparseTarget: WCHAR,
+ }
+
+ fn to_u16s<S: AsRef<OsStr>>(s: S) -> io::Result<Vec<u16>> {
+ Ok(s.as_ref().encode_wide().chain(Some(0)).collect())
+ }
+
+ // We're using low-level APIs to create the junction, and these are more
+ // picky about paths. For example, forward slashes cannot be used as a
+ // path separator, so we should try to canonicalize the path first.
+ let target = fs::canonicalize(target)?;
+
+ fs::create_dir(junction)?;
+
+ let path = to_u16s(junction)?;
+
+ unsafe {
+ let h = CreateFileW(
+ path.as_ptr(),
+ GENERIC_WRITE,
+ FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+ ptr::null_mut(),
+ OPEN_EXISTING,
+ FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS,
+ ptr::null_mut(),
+ );
+
+ let mut data = [0u8; MAXIMUM_REPARSE_DATA_BUFFER_SIZE as usize];
+ let db = data.as_mut_ptr() as *mut REPARSE_MOUNTPOINT_DATA_BUFFER;
+ let buf = &mut (*db).ReparseTarget as *mut u16;
+ let mut i = 0;
+ // FIXME: this conversion is very hacky
+ let v = br"\??\";
+ let v = v.iter().map(|x| *x as u16);
+ for c in v.chain(target.as_os_str().encode_wide().skip(4)) {
+ *buf.offset(i) = c;
+ i += 1;
+ }
+ *buf.offset(i) = 0;
+ i += 1;
+ (*db).ReparseTag = IO_REPARSE_TAG_MOUNT_POINT;
+ (*db).ReparseTargetMaximumLength = (i * 2) as WORD;
+ (*db).ReparseTargetLength = ((i - 1) * 2) as WORD;
+ (*db).ReparseDataLength = (*db).ReparseTargetLength as DWORD + 12;
+
+ let mut ret = 0;
+ let res = DeviceIoControl(
+ h as *mut _,
+ FSCTL_SET_REPARSE_POINT,
+ data.as_ptr() as *mut _,
+ (*db).ReparseDataLength + 8,
+ ptr::null_mut(),
+ 0,
+ &mut ret,
+ ptr::null_mut(),
+ );
+
+ let out = if res == 0 { Err(io::Error::last_os_error()) } else { Ok(()) };
+ CloseHandle(h);
+ out
+ }
+ }
+}
+
+/// The CI environment rustbuild is running in. This mainly affects how the logs
+/// are printed.
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+pub enum CiEnv {
+ /// Not a CI environment.
+ None,
+ /// The Azure Pipelines environment, for Linux (including Docker), Windows, and macOS builds.
+ AzurePipelines,
+ /// The GitHub Actions environment, for Linux (including Docker), Windows and macOS builds.
+ GitHubActions,
+}
+
+impl CiEnv {
+ /// Obtains the current CI environment.
+ pub fn current() -> CiEnv {
+ if env::var("TF_BUILD").map_or(false, |e| e == "True") {
+ CiEnv::AzurePipelines
+ } else if env::var("GITHUB_ACTIONS").map_or(false, |e| e == "true") {
+ CiEnv::GitHubActions
+ } else {
+ CiEnv::None
+ }
+ }
+
+ /// If in a CI environment, forces the command to run with colors.
+ pub fn force_coloring_in_ci(self, cmd: &mut Command) {
+ if self != CiEnv::None {
+ // Due to use of stamp/docker, the output stream of rustbuild is not
+ // a TTY in CI, so coloring is by-default turned off.
+ // The explicit `TERM=xterm` environment is needed for
+ // `--color always` to actually work. This env var was lost when
+ // compiling through the Makefile. Very strange.
+ cmd.env("TERM", "xterm").args(&["--color", "always"]);
+ }
+ }
+}
+
+pub fn forcing_clang_based_tests() -> bool {
+ if let Some(var) = env::var_os("RUSTBUILD_FORCE_CLANG_BASED_TESTS") {
+ match &var.to_string_lossy().to_lowercase()[..] {
+ "1" | "yes" | "on" => true,
+ "0" | "no" | "off" => false,
+ other => {
+ // Let's make sure typos don't go unnoticed
+ panic!(
+ "Unrecognized option '{}' set in \
+ RUSTBUILD_FORCE_CLANG_BASED_TESTS",
+ other
+ )
+ }
+ }
+ } else {
+ false
+ }
+}
+
+pub fn use_host_linker(target: TargetSelection) -> bool {
+ // FIXME: this information should be gotten by checking the linker flavor
+ // of the rustc target
+ !(target.contains("emscripten")
+ || target.contains("wasm32")
+ || target.contains("nvptx")
+ || target.contains("fortanix")
+ || target.contains("fuchsia")
+ || target.contains("bpf")
+ || target.contains("switch"))
+}
+
+pub fn is_valid_test_suite_arg<'a, P: AsRef<Path>>(
+ path: &'a Path,
+ suite_path: P,
+ builder: &Builder<'_>,
+) -> Option<&'a str> {
+ let suite_path = suite_path.as_ref();
+ let path = match path.strip_prefix(".") {
+ Ok(p) => p,
+ Err(_) => path,
+ };
+ if !path.starts_with(suite_path) {
+ return None;
+ }
+ let abs_path = builder.src.join(path);
+ let exists = abs_path.is_dir() || abs_path.is_file();
+ if !exists {
+ panic!(
+ "Invalid test suite filter \"{}\": file or directory does not exist",
+ abs_path.display()
+ );
+ }
+ // Since test suite paths are themselves directories, if we don't
+ // specify a directory or file, we'll get an empty string here
+ // (the result of the test suite directory without its suite prefix).
+ // Therefore, we need to filter these out, as only the first --test-args
+ // flag is respected, so providing an empty --test-args conflicts with
+ // any following it.
+ match path.strip_prefix(suite_path).ok().and_then(|p| p.to_str()) {
+ Some(s) if !s.is_empty() => Some(s),
+ _ => None,
+ }
+}
+
+pub fn run(cmd: &mut Command, print_cmd_on_fail: bool) {
+ if !try_run(cmd, print_cmd_on_fail) {
+ crate::detail_exit(1);
+ }
+}
+
+pub fn try_run(cmd: &mut Command, print_cmd_on_fail: bool) -> bool {
+ let status = match cmd.status() {
+ Ok(status) => status,
+ Err(e) => fail(&format!("failed to execute command: {:?}\nerror: {}", cmd, e)),
+ };
+ if !status.success() && print_cmd_on_fail {
+ println!(
+ "\n\ncommand did not execute successfully: {:?}\n\
+ expected success, got: {}\n\n",
+ cmd, status
+ );
+ }
+ status.success()
+}
+
+pub fn check_run(cmd: &mut Command, print_cmd_on_fail: bool) -> bool {
+ let status = match cmd.status() {
+ Ok(status) => status,
+ Err(e) => {
+ println!("failed to execute command: {:?}\nerror: {}", cmd, e);
+ return false;
+ }
+ };
+ if !status.success() && print_cmd_on_fail {
+ println!(
+ "\n\ncommand did not execute successfully: {:?}\n\
+ expected success, got: {}\n\n",
+ cmd, status
+ );
+ }
+ status.success()
+}
+
+pub fn run_suppressed(cmd: &mut Command) {
+ if !try_run_suppressed(cmd) {
+ crate::detail_exit(1);
+ }
+}
+
+pub fn try_run_suppressed(cmd: &mut Command) -> bool {
+ let output = match cmd.output() {
+ Ok(status) => status,
+ Err(e) => fail(&format!("failed to execute command: {:?}\nerror: {}", cmd, e)),
+ };
+ if !output.status.success() {
+ println!(
+ "\n\ncommand did not execute successfully: {:?}\n\
+ expected success, got: {}\n\n\
+ stdout ----\n{}\n\
+ stderr ----\n{}\n\n",
+ cmd,
+ output.status,
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ }
+ output.status.success()
+}
+
+pub fn make(host: &str) -> PathBuf {
+ if host.contains("dragonfly")
+ || host.contains("freebsd")
+ || host.contains("netbsd")
+ || host.contains("openbsd")
+ {
+ PathBuf::from("gmake")
+ } else {
+ PathBuf::from("make")
+ }
+}
+
+#[track_caller]
+pub fn output(cmd: &mut Command) -> String {
+ let output = match cmd.stderr(Stdio::inherit()).output() {
+ Ok(status) => status,
+ Err(e) => fail(&format!("failed to execute command: {:?}\nerror: {}", cmd, e)),
+ };
+ if !output.status.success() {
+ panic!(
+ "command did not execute successfully: {:?}\n\
+ expected success, got: {}",
+ cmd, output.status
+ );
+ }
+ String::from_utf8(output.stdout).unwrap()
+}
+
+/// Returns the last-modified time for `path`, or zero if it doesn't exist.
+pub fn mtime(path: &Path) -> SystemTime {
+ fs::metadata(path).and_then(|f| f.modified()).unwrap_or(UNIX_EPOCH)
+}
+
+/// Returns `true` if `dst` is up to date given that the file or files in `src`
+/// are used to generate it.
+///
+/// Uses last-modified time checks to verify this.
+pub fn up_to_date(src: &Path, dst: &Path) -> bool {
+ if !dst.exists() {
+ return false;
+ }
+ let threshold = mtime(dst);
+ let meta = match fs::metadata(src) {
+ Ok(meta) => meta,
+ Err(e) => panic!("source {:?} failed to get metadata: {}", src, e),
+ };
+ if meta.is_dir() {
+ dir_up_to_date(src, threshold)
+ } else {
+ meta.modified().unwrap_or(UNIX_EPOCH) <= threshold
+ }
+}
+
+fn dir_up_to_date(src: &Path, threshold: SystemTime) -> bool {
+ t!(fs::read_dir(src)).map(|e| t!(e)).all(|e| {
+ let meta = t!(e.metadata());
+ if meta.is_dir() {
+ dir_up_to_date(&e.path(), threshold)
+ } else {
+ meta.modified().unwrap_or(UNIX_EPOCH) < threshold
+ }
+ })
+}
+
+fn fail(s: &str) -> ! {
+ eprintln!("\n\n{}\n\n", s);
+ crate::detail_exit(1);
+}
+
+/// Copied from `std::path::absolute` until it stabilizes.
+///
+/// FIXME: this shouldn't exist.
+pub(crate) fn absolute(path: &Path) -> PathBuf {
+ if path.as_os_str().is_empty() {
+ panic!("can't make empty path absolute");
+ }
+ #[cfg(unix)]
+ {
+ t!(absolute_unix(path), format!("could not make path absolute: {}", path.display()))
+ }
+ #[cfg(windows)]
+ {
+ t!(absolute_windows(path), format!("could not make path absolute: {}", path.display()))
+ }
+ #[cfg(not(any(unix, windows)))]
+ {
+ println!("warning: bootstrap is not supported on non-unix platforms");
+ t!(std::fs::canonicalize(t!(std::env::current_dir()))).join(path)
+ }
+}
+
+#[cfg(unix)]
+/// Make a POSIX path absolute without changing its semantics.
+fn absolute_unix(path: &Path) -> io::Result<PathBuf> {
+ // This is mostly a wrapper around collecting `Path::components`, with
+ // exceptions made where this conflicts with the POSIX specification.
+ // See 4.13 Pathname Resolution, IEEE Std 1003.1-2017
+ // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13
+
+ use std::os::unix::prelude::OsStrExt;
+ let mut components = path.components();
+ let path_os = path.as_os_str().as_bytes();
+
+ let mut normalized = if path.is_absolute() {
+ // "If a pathname begins with two successive <slash> characters, the
+ // first component following the leading <slash> characters may be
+ // interpreted in an implementation-defined manner, although more than
+ // two leading <slash> characters shall be treated as a single <slash>
+ // character."
+ if path_os.starts_with(b"//") && !path_os.starts_with(b"///") {
+ components.next();
+ PathBuf::from("//")
+ } else {
+ PathBuf::new()
+ }
+ } else {
+ env::current_dir()?
+ };
+ normalized.extend(components);
+
+ // "Interfaces using pathname resolution may specify additional constraints
+ // when a pathname that does not name an existing directory contains at
+ // least one non- <slash> character and contains one or more trailing
+ // <slash> characters".
+ // A trailing <slash> is also meaningful if "a symbolic link is
+ // encountered during pathname resolution".
+
+ if path_os.ends_with(b"/") {
+ normalized.push("");
+ }
+
+ Ok(normalized)
+}
+
+#[cfg(windows)]
+fn absolute_windows(path: &std::path::Path) -> std::io::Result<std::path::PathBuf> {
+ use std::ffi::OsString;
+ use std::io::Error;
+ use std::os::windows::ffi::{OsStrExt, OsStringExt};
+ use std::ptr::null_mut;
+ #[link(name = "kernel32")]
+ extern "system" {
+ fn GetFullPathNameW(
+ lpFileName: *const u16,
+ nBufferLength: u32,
+ lpBuffer: *mut u16,
+ lpFilePart: *mut *const u16,
+ ) -> u32;
+ }
+
+ unsafe {
+ // encode the path as UTF-16
+ let path: Vec<u16> = path.as_os_str().encode_wide().chain([0]).collect();
+ let mut buffer = Vec::new();
+ // Loop until either success or failure.
+ loop {
+ // Try to get the absolute path
+ let len = GetFullPathNameW(
+ path.as_ptr(),
+ buffer.len().try_into().unwrap(),
+ buffer.as_mut_ptr(),
+ null_mut(),
+ );
+ match len as usize {
+ // Failure
+ 0 => return Err(Error::last_os_error()),
+ // Buffer is too small, resize.
+ len if len > buffer.len() => buffer.resize(len, 0),
+ // Success!
+ len => {
+ buffer.truncate(len);
+ return Ok(OsString::from_wide(&buffer).into());
+ }
+ }
+ }
+ }
+}
+
+/// Adapted from https://github.com/llvm/llvm-project/blob/782e91224601e461c019e0a4573bbccc6094fbcd/llvm/cmake/modules/HandleLLVMOptions.cmake#L1058-L1079
+///
+/// When `clang-cl` is used with instrumentation, we need to add clang's runtime library resource
+/// directory to the linker flags, otherwise there will be linker errors about the profiler runtime
+/// missing. This function returns the path to that directory.
+pub fn get_clang_cl_resource_dir(clang_cl_path: &str) -> PathBuf {
+ // Similar to how LLVM does it, to find clang's library runtime directory:
+ // - we ask `clang-cl` to locate the `clang_rt.builtins` lib.
+ let mut builtins_locator = Command::new(clang_cl_path);
+ builtins_locator.args(&["/clang:-print-libgcc-file-name", "/clang:--rtlib=compiler-rt"]);
+
+ let clang_rt_builtins = output(&mut builtins_locator);
+ let clang_rt_builtins = Path::new(clang_rt_builtins.trim());
+ assert!(
+ clang_rt_builtins.exists(),
+ "`clang-cl` must correctly locate the library runtime directory"
+ );
+
+ // - the profiler runtime will be located in the same directory as the builtins lib, like
+ // `$LLVM_DISTRO_ROOT/lib/clang/$LLVM_VERSION/lib/windows`.
+ let clang_rt_dir = clang_rt_builtins.parent().expect("The clang lib folder should exist");
+ clang_rt_dir.to_path_buf()
+}