diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-17 12:02:58 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-17 12:02:58 +0000 |
commit | 698f8c2f01ea549d77d7dc3338a12e04c11057b9 (patch) | |
tree | 173a775858bd501c378080a10dca74132f05bc50 /src/bootstrap | |
parent | Initial commit. (diff) | |
download | rustc-698f8c2f01ea549d77d7dc3338a12e04c11057b9.tar.xz rustc-698f8c2f01ea549d77d7dc3338a12e04c11057b9.zip |
Adding upstream version 1.64.0+dfsg1.upstream/1.64.0+dfsg1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
49 files changed, 24522 insertions, 0 deletions
diff --git a/src/bootstrap/CHANGELOG.md b/src/bootstrap/CHANGELOG.md new file mode 100644 index 000000000..85afc1f5f --- /dev/null +++ b/src/bootstrap/CHANGELOG.md @@ -0,0 +1,62 @@ +# Changelog + +All notable changes to bootstrap will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). + + +## [Changes since the last major version] + +- Vendoring is no longer done automatically when building from git sources. To use vendoring, run `cargo vendor` manually, or use the pre-vendored `rustc-src` tarball. +- `llvm-libunwind` now accepts `in-tree` (formerly true), `system` or `no` (formerly false) [#77703](https://github.com/rust-lang/rust/pull/77703) +- The options `infodir`, `localstatedir`, and `gpg-password-file` are no longer allowed in config.toml. Previously, they were ignored without warning. Note that `infodir` and `localstatedir` are still accepted by `./configure`, with a warning. [#82451](https://github.com/rust-lang/rust/pull/82451) +- Change the names for `dist` commands to match the component they generate. [#90684](https://github.com/rust-lang/rust/pull/90684) +- The `build.fast-submodules` option has been removed. Fast submodule checkouts are enabled unconditionally. Automatic submodule handling can still be disabled with `build.submodules = false`. +- Several unsupported `./configure` options have been removed: `optimize`, `parallel-compiler`. These can still be enabled with `--set`, although it isn't recommended. + +### Non-breaking changes + +- `x.py check` needs opt-in to check tests (--all-targets) [#77473](https://github.com/rust-lang/rust/pull/77473) +- The default bootstrap profiles are now located at `bootstrap/defaults/config.$PROFILE.toml` (previously they were located at `bootstrap/defaults/config.toml.$PROFILE`) [#77558](https://github.com/rust-lang/rust/pull/77558) +- If you have Rust already installed, `x.py` will now infer the host target + from the default rust toolchain. [#78513](https://github.com/rust-lang/rust/pull/78513) +- Add options for enabling overflow checks, one for std (`overflow-checks-std`) and one for everything else (`overflow-checks`). Both default to false. + + +## [Version 2] - 2020-09-25 + +- `host` now defaults to the value of `build` in all cases + + Previously `host` defaulted to an empty list when `target` was overridden, and to `build` otherwise + +### Non-breaking changes + +- Add `x.py setup` [#76631](https://github.com/rust-lang/rust/pull/76631) +- Add a changelog for x.py [#76626](https://github.com/rust-lang/rust/pull/76626) +- Optionally, download LLVM from CI on Linux and NixOS. This can be enabled with `download-ci-llvm = true` under `[llvm]`. + + [#76439](https://github.com/rust-lang/rust/pull/76349) + + [#76667](https://github.com/rust-lang/rust/pull/76667) + + [#76708](https://github.com/rust-lang/rust/pull/76708) +- Distribute rustc sources as part of `rustc-dev` [#76856](https://github.com/rust-lang/rust/pull/76856) +- Make the default stage for x.py configurable [#76625](https://github.com/rust-lang/rust/pull/76625). This can be enabled with `build-stage = N`, `doc-stage = N`, etc. +- Add a dedicated debug-logging option [#76588](https://github.com/rust-lang/rust/pull/76588). Previously, `debug-logging` could only be set with `debug-assertions`, slowing down the compiler more than necessary. +- Add sample defaults for x.py [#76628](https://github.com/rust-lang/rust/pull/76628) +- Add `--keep-stage-std`, which behaves like `keep-stage` but allows the stage + 0 compiler artifacts (i.e., stage1/bin/rustc) to be rebuilt if changed + [#77120](https://github.com/rust-lang/rust/pull/77120). + + +## [Version 1] - 2020-09-11 + +This is the first changelog entry, and it does not attempt to be an exhaustive list of features in x.py. +Instead, this documents the changes to bootstrap in the past 2 months. + +- Improve defaults in `x.py` [#73964](https://github.com/rust-lang/rust/pull/73964) + (see [blog post] for details) +- Set `ninja = true` by default [#74922](https://github.com/rust-lang/rust/pull/74922) +- Avoid trying to inversely cross-compile for build triple from host triples [#76415](https://github.com/rust-lang/rust/pull/76415) +- Allow blessing expect-tests in tools [#75975](https://github.com/rust-lang/rust/pull/75975) +- `x.py check` checks tests/examples/benches [#76258](https://github.com/rust-lang/rust/pull/76258) +- Fix `rust.use-lld` when linker is not set [#76326](https://github.com/rust-lang/rust/pull/76326) +- Build tests with LLD if `use-lld = true` was passed [#76378](https://github.com/rust-lang/rust/pull/76378) + +[blog post]: https://blog.rust-lang.org/inside-rust/2020/08/30/changes-to-x-py-defaults.html diff --git a/src/bootstrap/Cargo.lock b/src/bootstrap/Cargo.lock new file mode 100644 index 000000000..664ffa1dd --- /dev/null +++ b/src/bootstrap/Cargo.lock @@ -0,0 +1,778 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "block-buffer" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bootstrap" +version = "0.0.0" +dependencies = [ + "cc", + "cmake", + "fd-lock", + "filetime", + "getopts", + "hex", + "ignore", + "libc", + "num_cpus", + "once_cell", + "opener", + "pretty_assertions", + "serde", + "serde_json", + "sha2", + "sysinfo", + "tar", + "toml", + "walkdir", + "winapi", + "xz2", +] + +[[package]] +name = "bstr" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +dependencies = [ + "lazy_static", + "memchr", + "regex-automata", +] + +[[package]] +name = "cc" +version = "1.0.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cmake" +version = "0.1.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8ad8cef104ac57b68b89df3208164d228503abbdce70f6880ffa3d970e7443a" +dependencies = [ + "cc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + +[[package]] +name = "cpufeatures" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b" +dependencies = [ + "libc", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" +dependencies = [ + "cfg-if", + "lazy_static", +] + +[[package]] +name = "crypto-common" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "ctor" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f877be4f7c9f246b183111634f75baa039715e3f46ce860677d3b19a69fb229c" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "diff" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499" + +[[package]] +name = "digest" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "fd-lock" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11dcc7e4d79a8c89b9ab4c6f5c30b1fc4a83c420792da3542fd31179ed5f517" +dependencies = [ + "cfg-if", + "rustix", + "windows-sys", +] + +[[package]] +name = "filetime" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0408e2626025178a6a7f7ffc05a25bc47103229f19c113755de7bf63816290c" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "winapi", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "generic-array" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "globset" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10463d9ff00a2a068db14231982f5132edebad0d7660cd956a1c30292dbcbfbd" +dependencies = [ + "aho-corasick", + "bstr", + "fnv", + "log", + "regex", +] + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "ignore" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "713f1b139373f96a2e0ce3ac931cd01ee973c3c5dd7c40c0c2efe96ad2b6751d" +dependencies = [ + "crossbeam-utils", + "globset", + "lazy_static", + "log", + "memchr", + "regex", + "same-file", + "thread_local", + "walkdir", + "winapi-util", +] + +[[package]] +name = "io-lifetimes" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24c3f4eff5495aee4c0399d7b6a0dc2b6e81be84242ffbfcf253ebacccc1d0cb" + +[[package]] +name = "itoa" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" + +[[package]] +name = "linux-raw-sys" +version = "0.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d" + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "lzma-sys" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb4b7c3eddad11d3af9e86c487607d2d2442d185d848575365c4856ba96d619" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + +[[package]] +name = "ntapi" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" +dependencies = [ + "winapi", +] + +[[package]] +name = "num_cpus" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225" + +[[package]] +name = "opener" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea3ebcd72a54701f56345f16785a6d3ac2df7e986d273eb4395c0b01db17952" +dependencies = [ + "bstr", + "winapi", +] + +[[package]] +name = "output_vt100" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "628223faebab4e3e40667ee0b2336d34a5b960ff60ea743ddfdbcf7770bcfb66" +dependencies = [ + "winapi", +] + +[[package]] +name = "pkg-config" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" + +[[package]] +name = "pretty_assertions" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cab0e7c02cf376875e9335e0ba1da535775beb5450d21e1dffca068818ed98b" +dependencies = [ + "ansi_term", + "ctor", + "diff", + "output_vt100", +] + +[[package]] +name = "proc-macro2" +version = "1.0.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rayon" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + +[[package]] +name = "redox_syscall" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" + +[[package]] +name = "regex-syntax" +version = "0.6.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64" + +[[package]] +name = "rustix" +version = "0.35.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef258c11e17f5c01979a10543a30a4e12faef6aab217a74266e747eefa3aed88" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "ryu" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "serde" +version = "1.0.137" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.137" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "syn" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbaf6116ab8924f39d52792136fb74fd60a80194cf1b1c6ffa6453eef1c3f942" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sysinfo" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2809487b962344ca55d9aea565f9ffbcb6929780802217acc82561f6746770" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "rayon", + "winapi", +] + +[[package]] +name = "tar" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "thread_local" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +dependencies = [ + "once_cell", +] + +[[package]] +name = "toml" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +dependencies = [ + "serde", +] + +[[package]] +name = "typenum" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" + +[[package]] +name = "unicode-ident" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee" + +[[package]] +name = "unicode-width" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" + +[[package]] +name = "windows_i686_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" + +[[package]] +name = "windows_i686_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" + +[[package]] +name = "xattr" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc" +dependencies = [ + "libc", +] + +[[package]] +name = "xz2" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c179869f34fc7c01830d3ce7ea2086bc3a07e0d35289b667d0a8bf910258926c" +dependencies = [ + "lzma-sys", +] diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml new file mode 100644 index 000000000..84f6aaf99 --- /dev/null +++ b/src/bootstrap/Cargo.toml @@ -0,0 +1,85 @@ +[package] +name = "bootstrap" +version = "0.0.0" +edition = "2021" +build = "build.rs" +default-run = "bootstrap" + +[lib] +path = "lib.rs" +doctest = false + +[[bin]] +name = "bootstrap" +path = "bin/main.rs" +test = false + +[[bin]] +name = "rustc" +path = "bin/rustc.rs" +test = false + +[[bin]] +name = "rustdoc" +path = "bin/rustdoc.rs" +test = false + +[[bin]] +name = "sccache-plus-cl" +path = "bin/sccache-plus-cl.rs" +test = false + +[[bin]] +name = "llvm-config-wrapper" +path = "bin/llvm-config-wrapper.rs" +test = false + +[dependencies] +cmake = "0.1.38" +fd-lock = "3.0.6" +filetime = "0.2" +num_cpus = "1.0" +getopts = "0.2.19" +cc = "1.0.69" +libc = "0.2" +hex = "0.4" +serde = { version = "1.0.8", features = ["derive"] } +serde_json = "1.0.2" +sha2 = "0.10" +tar = "0.4" +toml = "0.5" +ignore = "0.4.10" +opener = "0.5" +once_cell = "1.7.2" +xz2 = "0.1" +walkdir = "2" + +# Dependencies needed by the build-metrics feature +sysinfo = { version = "0.24.1", optional = true } + +[target.'cfg(windows)'.dependencies.winapi] +version = "0.3" +features = [ + "fileapi", + "ioapiset", + "jobapi2", + "handleapi", + "winioctl", + "psapi", + "impl-default", + "timezoneapi", +] + +[dev-dependencies] +pretty_assertions = "0.7" + +[features] +build-metrics = ["sysinfo"] + +# We care a lot about bootstrap's compile times, so don't include debuginfo for +# dependencies, only bootstrap itself. +[profile.dev] +debug = 0 +[profile.dev.package] +# Only use debuginfo=1 to further reduce compile times. +bootstrap.debug = 1 diff --git a/src/bootstrap/README.md b/src/bootstrap/README.md new file mode 100644 index 000000000..a2e596bf4 --- /dev/null +++ b/src/bootstrap/README.md @@ -0,0 +1,333 @@ +# rustbuild - Bootstrapping Rust + +This is an in-progress README which is targeted at helping to explain how Rust +is bootstrapped and in general some of the technical details of the build +system. + +## Using rustbuild + +The rustbuild build system has a primary entry point, a top level `x.py` script: + +```sh +$ python ./x.py build +``` + +Note that if you're on Unix you should be able to execute the script directly: + +```sh +$ ./x.py build +``` + +The script accepts commands, flags, and arguments to determine what to do: + +* `build` - a general purpose command for compiling code. Alone `build` will + bootstrap the entire compiler, and otherwise arguments passed indicate what to + build. For example: + + ``` + # build the whole compiler + ./x.py build --stage 2 + + # build the stage1 compiler + ./x.py build + + # build stage0 libstd + ./x.py build --stage 0 library/std + + # build a particular crate in stage0 + ./x.py build --stage 0 library/test + ``` + + If files are dirty that would normally be rebuilt from stage 0, that can be + overridden using `--keep-stage 0`. Using `--keep-stage n` will skip all steps + that belong to stage n or earlier: + + ``` + # build stage 1, keeping old build products for stage 0 + ./x.py build --keep-stage 0 + ``` + +* `test` - a command for executing unit tests. Like the `build` command this + will execute the entire test suite by default, and otherwise it can be used to + select which test suite is run: + + ``` + # run all unit tests + ./x.py test + + # execute tool tests + ./x.py test tidy + + # execute the UI test suite + ./x.py test src/test/ui + + # execute only some tests in the UI test suite + ./x.py test src/test/ui --test-args substring-of-test-name + + # execute tests in the standard library in stage0 + ./x.py test --stage 0 library/std + + # execute tests in the core and standard library in stage0, + # without running doc tests (thus avoid depending on building the compiler) + ./x.py test --stage 0 --no-doc library/core library/std + + # execute all doc tests + ./x.py test src/doc + ``` + +* `doc` - a command for building documentation. Like above can take arguments + for what to document. + +## Configuring rustbuild + +There are currently two methods for configuring the rustbuild build system. + +First, rustbuild offers a TOML-based configuration system with a `config.toml` +file. An example of this configuration can be found at `config.toml.example`, +and the configuration file can also be passed as `--config path/to/config.toml` +if the build system is being invoked manually (via the python script). + +Next, the `./configure` options serialized in `config.mk` will be +parsed and read. That is, if any `./configure` options are passed, they'll be +handled naturally. `./configure` should almost never be used for local +installations, and is primarily useful for CI. Prefer to customize behavior +using `config.toml`. + +Finally, rustbuild makes use of the [cc-rs crate] which has [its own +method][env-vars] of configuring C compilers and C flags via environment +variables. + +[cc-rs crate]: https://github.com/alexcrichton/cc-rs +[env-vars]: https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables + +## Build stages + +The rustbuild build system goes through a few phases to actually build the +compiler. What actually happens when you invoke rustbuild is: + +1. The entry point script, `x.py` is run. This script is + responsible for downloading the stage0 compiler/Cargo binaries, and it then + compiles the build system itself (this folder). Finally, it then invokes the + actual `bootstrap` binary build system. +2. In Rust, `bootstrap` will slurp up all configuration, perform a number of + sanity checks (compilers exist for example), and then start building the + stage0 artifacts. +3. The stage0 `cargo` downloaded earlier is used to build the standard library + and the compiler, and then these binaries are then copied to the `stage1` + directory. That compiler is then used to generate the stage1 artifacts which + are then copied to the stage2 directory, and then finally the stage2 + artifacts are generated using that compiler. + +The goal of each stage is to (a) leverage Cargo as much as possible and failing +that (b) leverage Rust as much as possible! + +## Incremental builds + +You can configure rustbuild to use incremental compilation with the +`--incremental` flag: + +```sh +$ ./x.py build --incremental +``` + +The `--incremental` flag will store incremental compilation artifacts +in `build/<host>/stage0-incremental`. Note that we only use incremental +compilation for the stage0 -> stage1 compilation -- this is because +the stage1 compiler is changing, and we don't try to cache and reuse +incremental artifacts across different versions of the compiler. + +You can always drop the `--incremental` to build as normal (but you +will still be using the local nightly as your bootstrap). + +## Directory Layout + +This build system houses all output under the `build` directory, which looks +like this: + +```sh +# Root folder of all output. Everything is scoped underneath here +build/ + + # Location where the stage0 compiler downloads are all cached. This directory + # only contains the tarballs themselves as they're extracted elsewhere. + cache/ + 2015-12-19/ + 2016-01-15/ + 2016-01-21/ + ... + + # Output directory for building this build system itself. The stage0 + # cargo/rustc are used to build the build system into this location. + bootstrap/ + debug/ + release/ + + # Output of the dist-related steps like dist-std, dist-rustc, and dist-docs + dist/ + + # Temporary directory used for various input/output as part of various stages + tmp/ + + # Each remaining directory is scoped by the "host" triple of compilation at + # hand. + x86_64-unknown-linux-gnu/ + + # The build artifacts for the `compiler-rt` library for the target this + # folder is under. The exact layout here will likely depend on the platform, + # and this is also built with CMake so the build system is also likely + # different. + compiler-rt/ + build/ + + # Output folder for LLVM if it is compiled for this target + llvm/ + + # build folder (e.g. the platform-specific build system). Like with + # compiler-rt this is compiled with CMake + build/ + + # Installation of LLVM. Note that we run the equivalent of 'make install' + # for LLVM to setup these folders. + bin/ + lib/ + include/ + share/ + ... + + # Output folder for all documentation of this target. This is what's filled + # in whenever the `doc` step is run. + doc/ + + # Output for all compiletest-based test suites + test/ + ui/ + debuginfo/ + ... + + # Location where the stage0 Cargo and Rust compiler are unpacked. This + # directory is purely an extracted and overlaid tarball of these two (done + # by the bootstrapy python script). In theory the build system does not + # modify anything under this directory afterwards. + stage0/ + + # These to build directories are the cargo output directories for builds of + # the standard library and compiler, respectively. Internally these may also + # have other target directories, which represent artifacts being compiled + # from the host to the specified target. + # + # Essentially, each of these directories is filled in by one `cargo` + # invocation. The build system instruments calling Cargo in the right order + # with the right variables to ensure these are filled in correctly. + stageN-std/ + stageN-test/ + stageN-rustc/ + stageN-tools/ + + # This is a special case of the above directories, **not** filled in via + # Cargo but rather the build system itself. The stage0 compiler already has + # a set of target libraries for its own host triple (in its own sysroot) + # inside of stage0/. When we run the stage0 compiler to bootstrap more + # things, however, we don't want to use any of these libraries (as those are + # the ones that we're building). So essentially, when the stage1 compiler is + # being compiled (e.g. after libstd has been built), *this* is used as the + # sysroot for the stage0 compiler being run. + # + # Basically this directory is just a temporary artifact use to configure the + # stage0 compiler to ensure that the libstd we just built is used to + # compile the stage1 compiler. + stage0-sysroot/lib/ + + # These output directories are intended to be standalone working + # implementations of the compiler (corresponding to each stage). The build + # system will link (using hard links) output from stageN-{std,rustc} into + # each of these directories. + # + # In theory there is no extra build output in these directories. + stage1/ + stage2/ + stage3/ +``` + +## Cargo projects + +The current build is unfortunately not quite as simple as `cargo build` in a +directory, but rather the compiler is split into three different Cargo projects: + +* `library/std` - the standard library +* `library/test` - testing support, depends on libstd +* `compiler/rustc` - the actual compiler itself + +Each "project" has a corresponding Cargo.lock file with all dependencies, and +this means that building the compiler involves running Cargo three times. The +structure here serves two goals: + +1. Facilitating dependencies coming from crates.io. These dependencies don't + depend on `std`, so libstd is a separate project compiled ahead of time + before the actual compiler builds. +2. Splitting "host artifacts" from "target artifacts". That is, when building + code for an arbitrary target you don't need the entire compiler, but you'll + end up needing libraries like libtest that depend on std but also want to use + crates.io dependencies. Hence, libtest is split out as its own project that + is sequenced after `std` but before `rustc`. This project is built for all + targets. + +There is some loss in build parallelism here because libtest can be compiled in +parallel with a number of rustc artifacts, but in theory the loss isn't too bad! + +## Build tools + +We've actually got quite a few tools that we use in the compiler's build system +and for testing. To organize these, each tool is a project in `src/tools` with a +corresponding `Cargo.toml`. All tools are compiled with Cargo (currently having +independent `Cargo.lock` files) and do not currently explicitly depend on the +compiler or standard library. Compiling each tool is sequenced after the +appropriate libstd/libtest/librustc compile above. + +## Extending rustbuild + +So you'd like to add a feature to the rustbuild build system or just fix a bug. +Great! One of the major motivational factors for moving away from `make` is that +Rust is in theory much easier to read, modify, and write. If you find anything +excessively confusing, please open an issue on this and we'll try to get it +documented or simplified pronto. + +First up, you'll probably want to read over the documentation above as that'll +give you a high level overview of what rustbuild is doing. You also probably +want to play around a bit yourself by just getting it up and running before you +dive too much into the actual build system itself. + +After that, each module in rustbuild should have enough documentation to keep +you up and running. Some general areas that you may be interested in modifying +are: + +* Adding a new build tool? Take a look at `bootstrap/tool.rs` for examples of + other tools. +* Adding a new compiler crate? Look no further! Adding crates can be done by + adding a new directory with `Cargo.toml` followed by configuring all + `Cargo.toml` files accordingly. +* Adding a new dependency from crates.io? This should just work inside the + compiler artifacts stage (everything other than libtest and libstd). +* Adding a new configuration option? You'll want to modify `bootstrap/flags.rs` + for command line flags and then `bootstrap/config.rs` to copy the flags to the + `Config` struct. +* Adding a sanity check? Take a look at `bootstrap/sanity.rs`. + +If you make a major change, please remember to: + ++ Update `VERSION` in `src/bootstrap/main.rs`. +* Update `changelog-seen = N` in `config.toml.example`. +* Add an entry in `src/bootstrap/CHANGELOG.md`. + +A 'major change' includes + +* A new option or +* A change in the default options. + +Changes that do not affect contributors to the compiler or users +building rustc from source don't need an update to `VERSION`. + +If you have any questions feel free to reach out on the `#t-infra` channel in +the [Rust Zulip server][rust-zulip] or ask on internals.rust-lang.org. When +you encounter bugs, please file issues on the rust-lang/rust issue tracker. + +[rust-zulip]: https://rust-lang.zulipchat.com/#narrow/stream/242791-t-infra diff --git a/src/bootstrap/bin/llvm-config-wrapper.rs b/src/bootstrap/bin/llvm-config-wrapper.rs new file mode 100644 index 000000000..89984bb55 --- /dev/null +++ b/src/bootstrap/bin/llvm-config-wrapper.rs @@ -0,0 +1,24 @@ +// The sheer existence of this file is an awful hack. See the comments in +// `src/bootstrap/native.rs` for why this is needed when compiling LLD. + +use std::env; +use std::io::{self, Write}; +use std::process::{self, Command, Stdio}; + +fn main() { + let real_llvm_config = env::var_os("LLVM_CONFIG_REAL").unwrap(); + let mut cmd = Command::new(real_llvm_config); + cmd.args(env::args().skip(1)).stderr(Stdio::piped()); + let output = cmd.output().expect("failed to spawn llvm-config"); + let mut stdout = String::from_utf8_lossy(&output.stdout); + + if let Ok(to_replace) = env::var("LLVM_CONFIG_SHIM_REPLACE") { + if let Ok(replace_with) = env::var("LLVM_CONFIG_SHIM_REPLACE_WITH") { + stdout = stdout.replace(&to_replace, &replace_with).into(); + } + } + + print!("{}", stdout.replace("\\", "/")); + io::stdout().flush().unwrap(); + process::exit(output.status.code().unwrap_or(1)); +} diff --git a/src/bootstrap/bin/main.rs b/src/bootstrap/bin/main.rs new file mode 100644 index 000000000..9b4861ccd --- /dev/null +++ b/src/bootstrap/bin/main.rs @@ -0,0 +1,100 @@ +//! rustbuild, the Rust build system +//! +//! This is the entry point for the build system used to compile the `rustc` +//! compiler. Lots of documentation can be found in the `README.md` file in the +//! parent directory, and otherwise documentation can be found throughout the `build` +//! directory in each respective module. + +use std::env; + +use bootstrap::{t, Build, Config, Subcommand, VERSION}; + +fn main() { + let args = env::args().skip(1).collect::<Vec<_>>(); + let config = Config::parse(&args); + + let mut build_lock; + let _build_lock_guard; + if cfg!(any(unix, windows)) { + build_lock = fd_lock::RwLock::new(t!(std::fs::File::create(config.out.join("lock")))); + _build_lock_guard = match build_lock.try_write() { + Ok(lock) => lock, + err => { + println!("warning: build directory locked, waiting for lock"); + drop(err); + t!(build_lock.write()) + } + }; + } else { + println!("warning: file locking not supported for target, not locking build directory"); + } + + // check_version warnings are not printed during setup + let changelog_suggestion = + if matches!(config.cmd, Subcommand::Setup { .. }) { None } else { check_version(&config) }; + + // NOTE: Since `./configure` generates a `config.toml`, distro maintainers will see the + // changelog warning, not the `x.py setup` message. + let suggest_setup = !config.config.exists() && !matches!(config.cmd, Subcommand::Setup { .. }); + if suggest_setup { + println!("warning: you have not made a `config.toml`"); + println!( + "help: consider running `./x.py setup` or copying `config.toml.example` by running \ + `cp config.toml.example config.toml`" + ); + } else if let Some(suggestion) = &changelog_suggestion { + println!("{}", suggestion); + } + + let pre_commit = config.src.join(".git").join("hooks").join("pre-commit"); + Build::new(config).build(); + + if suggest_setup { + println!("warning: you have not made a `config.toml`"); + println!( + "help: consider running `./x.py setup` or copying `config.toml.example` by running \ + `cp config.toml.example config.toml`" + ); + } else if let Some(suggestion) = &changelog_suggestion { + println!("{}", suggestion); + } + + // Give a warning if the pre-commit script is in pre-commit and not pre-push. + // HACK: Since the commit script uses hard links, we can't actually tell if it was installed by x.py setup or not. + // We could see if it's identical to src/etc/pre-push.sh, but pre-push may have been modified in the meantime. + // Instead, look for this comment, which is almost certainly not in any custom hook. + if std::fs::read_to_string(pre_commit).map_or(false, |contents| { + contents.contains("https://github.com/rust-lang/rust/issues/77620#issuecomment-705144570") + }) { + println!( + "warning: You have the pre-push script installed to .git/hooks/pre-commit. \ + Consider moving it to .git/hooks/pre-push instead, which runs less often." + ); + } + + if suggest_setup || changelog_suggestion.is_some() { + println!("note: this message was printed twice to make it more likely to be seen"); + } +} + +fn check_version(config: &Config) -> Option<String> { + let mut msg = String::new(); + + let suggestion = if let Some(seen) = config.changelog_seen { + if seen != VERSION { + msg.push_str("warning: there have been changes to x.py since you last updated.\n"); + format!("update `config.toml` to use `changelog-seen = {}` instead", VERSION) + } else { + return None; + } + } else { + msg.push_str("warning: x.py has made several changes recently you may want to look at\n"); + format!("add `changelog-seen = {}` at the top of `config.toml`", VERSION) + }; + + msg.push_str("help: consider looking at the changes in `src/bootstrap/CHANGELOG.md`\n"); + msg.push_str("note: to silence this warning, "); + msg.push_str(&suggestion); + + Some(msg) +} diff --git a/src/bootstrap/bin/rustc.rs b/src/bootstrap/bin/rustc.rs new file mode 100644 index 000000000..40a3cc6d1 --- /dev/null +++ b/src/bootstrap/bin/rustc.rs @@ -0,0 +1,369 @@ +//! Shim which is passed to Cargo as "rustc" when running the bootstrap. +//! +//! This shim will take care of some various tasks that our build process +//! requires that Cargo can't quite do through normal configuration: +//! +//! 1. When compiling build scripts and build dependencies, we need a guaranteed +//! full standard library available. The only compiler which actually has +//! this is the snapshot, so we detect this situation and always compile with +//! the snapshot compiler. +//! 2. We pass a bunch of `--cfg` and other flags based on what we're compiling +//! (and this slightly differs based on a whether we're using a snapshot or +//! not), so we do that all here. +//! +//! This may one day be replaced by RUSTFLAGS, but the dynamic nature of +//! switching compilers for the bootstrap and for build scripts will probably +//! never get replaced. + +include!("../dylib_util.rs"); + +use std::env; +use std::path::PathBuf; +use std::process::{Child, Command}; +use std::str::FromStr; +use std::time::Instant; + +fn main() { + let args = env::args_os().skip(1).collect::<Vec<_>>(); + + // Detect whether or not we're a build script depending on whether --target + // is passed (a bit janky...) + let target = args.windows(2).find(|w| &*w[0] == "--target").and_then(|w| w[1].to_str()); + let version = args.iter().find(|w| &**w == "-vV"); + + let verbose = match env::var("RUSTC_VERBOSE") { + Ok(s) => usize::from_str(&s).expect("RUSTC_VERBOSE should be an integer"), + Err(_) => 0, + }; + + // Use a different compiler for build scripts, since there may not yet be a + // libstd for the real compiler to use. However, if Cargo is attempting to + // determine the version of the compiler, the real compiler needs to be + // used. Currently, these two states are differentiated based on whether + // --target and -vV is/isn't passed. + let (rustc, libdir) = if target.is_none() && version.is_none() { + ("RUSTC_SNAPSHOT", "RUSTC_SNAPSHOT_LIBDIR") + } else { + ("RUSTC_REAL", "RUSTC_LIBDIR") + }; + let stage = env::var("RUSTC_STAGE").expect("RUSTC_STAGE was not set"); + let sysroot = env::var_os("RUSTC_SYSROOT").expect("RUSTC_SYSROOT was not set"); + let on_fail = env::var_os("RUSTC_ON_FAIL").map(Command::new); + + let rustc = env::var_os(rustc).unwrap_or_else(|| panic!("{:?} was not set", rustc)); + let libdir = env::var_os(libdir).unwrap_or_else(|| panic!("{:?} was not set", libdir)); + let mut dylib_path = dylib_path(); + dylib_path.insert(0, PathBuf::from(&libdir)); + + let mut cmd = Command::new(rustc); + cmd.args(&args).env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); + + // Get the name of the crate we're compiling, if any. + let crate_name = + args.windows(2).find(|args| args[0] == "--crate-name").and_then(|args| args[1].to_str()); + + if let Some(crate_name) = crate_name { + if let Some(target) = env::var_os("RUSTC_TIME") { + if target == "all" + || target.into_string().unwrap().split(',').any(|c| c.trim() == crate_name) + { + cmd.arg("-Ztime"); + } + } + } + + // Print backtrace in case of ICE + if env::var("RUSTC_BACKTRACE_ON_ICE").is_ok() && env::var("RUST_BACKTRACE").is_err() { + cmd.env("RUST_BACKTRACE", "1"); + } + + if let Ok(lint_flags) = env::var("RUSTC_LINT_FLAGS") { + cmd.args(lint_flags.split_whitespace()); + } + + if target.is_some() { + // The stage0 compiler has a special sysroot distinct from what we + // actually downloaded, so we just always pass the `--sysroot` option, + // unless one is already set. + if !args.iter().any(|arg| arg == "--sysroot") { + cmd.arg("--sysroot").arg(&sysroot); + } + + // If we're compiling specifically the `panic_abort` crate then we pass + // the `-C panic=abort` option. Note that we do not do this for any + // other crate intentionally as this is the only crate for now that we + // ship with panic=abort. + // + // This... is a bit of a hack how we detect this. Ideally this + // information should be encoded in the crate I guess? Would likely + // require an RFC amendment to RFC 1513, however. + // + // `compiler_builtins` are unconditionally compiled with panic=abort to + // workaround undefined references to `rust_eh_unwind_resume` generated + // otherwise, see issue https://github.com/rust-lang/rust/issues/43095. + if crate_name == Some("panic_abort") + || crate_name == Some("compiler_builtins") && stage != "0" + { + cmd.arg("-C").arg("panic=abort"); + } + } else { + // FIXME(rust-lang/cargo#5754) we shouldn't be using special env vars + // here, but rather Cargo should know what flags to pass rustc itself. + + // Override linker if necessary. + if let Ok(host_linker) = env::var("RUSTC_HOST_LINKER") { + cmd.arg(format!("-Clinker={}", host_linker)); + } + if env::var_os("RUSTC_HOST_FUSE_LD_LLD").is_some() { + cmd.arg("-Clink-args=-fuse-ld=lld"); + } + + if let Ok(s) = env::var("RUSTC_HOST_CRT_STATIC") { + if s == "true" { + cmd.arg("-C").arg("target-feature=+crt-static"); + } + if s == "false" { + cmd.arg("-C").arg("target-feature=-crt-static"); + } + } + + // Cargo doesn't pass RUSTFLAGS to proc_macros: + // https://github.com/rust-lang/cargo/issues/4423 + // Thus, if we are on stage 0, we explicitly set `--cfg=bootstrap`. + // We also declare that the flag is expected, which is mainly needed for + // later stages so that they don't warn about #[cfg(bootstrap)], + // but enabling it for stage 0 too lets any warnings, if they occur, + // occur more early on, e.g. about #[cfg(bootstrap = "foo")]. + if stage == "0" { + cmd.arg("--cfg=bootstrap"); + } + cmd.arg("-Zunstable-options"); + cmd.arg("--check-cfg=values(bootstrap)"); + } + + if let Ok(map) = env::var("RUSTC_DEBUGINFO_MAP") { + cmd.arg("--remap-path-prefix").arg(&map); + } + + // Force all crates compiled by this compiler to (a) be unstable and (b) + // allow the `rustc_private` feature to link to other unstable crates + // also in the sysroot. We also do this for host crates, since those + // may be proc macros, in which case we might ship them. + if env::var_os("RUSTC_FORCE_UNSTABLE").is_some() && (stage != "0" || target.is_some()) { + cmd.arg("-Z").arg("force-unstable-if-unmarked"); + } + + if let Ok(flags) = env::var("MAGIC_EXTRA_RUSTFLAGS") { + for flag in flags.split(' ') { + cmd.arg(flag); + } + } + + let is_test = args.iter().any(|a| a == "--test"); + if verbose > 2 { + let rust_env_vars = + env::vars().filter(|(k, _)| k.starts_with("RUST") || k.starts_with("CARGO")); + let prefix = if is_test { "[RUSTC-SHIM] rustc --test" } else { "[RUSTC-SHIM] rustc" }; + let prefix = match crate_name { + Some(crate_name) => format!("{} {}", prefix, crate_name), + None => prefix.to_string(), + }; + for (i, (k, v)) in rust_env_vars.enumerate() { + eprintln!("{} env[{}]: {:?}={:?}", prefix, i, k, v); + } + eprintln!("{} working directory: {}", prefix, env::current_dir().unwrap().display()); + eprintln!( + "{} command: {:?}={:?} {:?}", + prefix, + dylib_path_var(), + env::join_paths(&dylib_path).unwrap(), + cmd, + ); + eprintln!("{} sysroot: {:?}", prefix, sysroot); + eprintln!("{} libdir: {:?}", prefix, libdir); + } + + let start = Instant::now(); + let (child, status) = { + let errmsg = format!("\nFailed to run:\n{:?}\n-------------", cmd); + let mut child = cmd.spawn().expect(&errmsg); + let status = child.wait().expect(&errmsg); + (child, status) + }; + + if env::var_os("RUSTC_PRINT_STEP_TIMINGS").is_some() + || env::var_os("RUSTC_PRINT_STEP_RUSAGE").is_some() + { + if let Some(crate_name) = crate_name { + let dur = start.elapsed(); + // If the user requested resource usage data, then + // include that in addition to the timing output. + let rusage_data = + env::var_os("RUSTC_PRINT_STEP_RUSAGE").and_then(|_| format_rusage_data(child)); + eprintln!( + "[RUSTC-TIMING] {} test:{} {}.{:03}{}{}", + crate_name, + is_test, + dur.as_secs(), + dur.subsec_millis(), + if rusage_data.is_some() { " " } else { "" }, + rusage_data.unwrap_or(String::new()), + ); + } + } + + if status.success() { + std::process::exit(0); + // note: everything below here is unreachable. do not put code that + // should run on success, after this block. + } + if verbose > 0 { + println!("\nDid not run successfully: {}\n{:?}\n-------------", status, cmd); + } + + if let Some(mut on_fail) = on_fail { + on_fail.status().expect("Could not run the on_fail command"); + } + + // Preserve the exit code. In case of signal, exit with 0xfe since it's + // awkward to preserve this status in a cross-platform way. + match status.code() { + Some(i) => std::process::exit(i), + None => { + eprintln!("rustc exited with {}", status); + std::process::exit(0xfe); + } + } +} + +#[cfg(all(not(unix), not(windows)))] +// In the future we can add this for more platforms +fn format_rusage_data(_child: Child) -> Option<String> { + None +} + +#[cfg(windows)] +fn format_rusage_data(child: Child) -> Option<String> { + use std::os::windows::io::AsRawHandle; + use winapi::um::{processthreadsapi, psapi, timezoneapi}; + let handle = child.as_raw_handle(); + macro_rules! try_bool { + ($e:expr) => { + if $e != 1 { + return None; + } + }; + } + + let mut user_filetime = Default::default(); + let mut user_time = Default::default(); + let mut kernel_filetime = Default::default(); + let mut kernel_time = Default::default(); + let mut memory_counters = psapi::PROCESS_MEMORY_COUNTERS::default(); + + unsafe { + try_bool!(processthreadsapi::GetProcessTimes( + handle, + &mut Default::default(), + &mut Default::default(), + &mut kernel_filetime, + &mut user_filetime, + )); + try_bool!(timezoneapi::FileTimeToSystemTime(&user_filetime, &mut user_time)); + try_bool!(timezoneapi::FileTimeToSystemTime(&kernel_filetime, &mut kernel_time)); + + // Unlike on Linux with RUSAGE_CHILDREN, this will only return memory information for the process + // with the given handle and none of that process's children. + try_bool!(psapi::GetProcessMemoryInfo( + handle as _, + &mut memory_counters as *mut _ as _, + std::mem::size_of::<psapi::PROCESS_MEMORY_COUNTERS_EX>() as u32, + )); + } + + // Guide on interpreting these numbers: + // https://docs.microsoft.com/en-us/windows/win32/psapi/process-memory-usage-information + let peak_working_set = memory_counters.PeakWorkingSetSize / 1024; + let peak_page_file = memory_counters.PeakPagefileUsage / 1024; + let peak_paged_pool = memory_counters.QuotaPeakPagedPoolUsage / 1024; + let peak_nonpaged_pool = memory_counters.QuotaPeakNonPagedPoolUsage / 1024; + Some(format!( + "user: {USER_SEC}.{USER_USEC:03} \ + sys: {SYS_SEC}.{SYS_USEC:03} \ + peak working set (kb): {PEAK_WORKING_SET} \ + peak page file usage (kb): {PEAK_PAGE_FILE} \ + peak paged pool usage (kb): {PEAK_PAGED_POOL} \ + peak non-paged pool usage (kb): {PEAK_NONPAGED_POOL} \ + page faults: {PAGE_FAULTS}", + USER_SEC = user_time.wSecond + (user_time.wMinute * 60), + USER_USEC = user_time.wMilliseconds, + SYS_SEC = kernel_time.wSecond + (kernel_time.wMinute * 60), + SYS_USEC = kernel_time.wMilliseconds, + PEAK_WORKING_SET = peak_working_set, + PEAK_PAGE_FILE = peak_page_file, + PEAK_PAGED_POOL = peak_paged_pool, + PEAK_NONPAGED_POOL = peak_nonpaged_pool, + PAGE_FAULTS = memory_counters.PageFaultCount, + )) +} + +#[cfg(unix)] +/// Tries to build a string with human readable data for several of the rusage +/// fields. Note that we are focusing mainly on data that we believe to be +/// supplied on Linux (the `rusage` struct has other fields in it but they are +/// currently unsupported by Linux). +fn format_rusage_data(_child: Child) -> Option<String> { + let rusage: libc::rusage = unsafe { + let mut recv = std::mem::zeroed(); + // -1 is RUSAGE_CHILDREN, which means to get the rusage for all children + // (and grandchildren, etc) processes that have respectively terminated + // and been waited for. + let retval = libc::getrusage(-1, &mut recv); + if retval != 0 { + return None; + } + recv + }; + // Mac OS X reports the maxrss in bytes, not kb. + let divisor = if env::consts::OS == "macos" { 1024 } else { 1 }; + let maxrss = (rusage.ru_maxrss + (divisor - 1)) / divisor; + + let mut init_str = format!( + "user: {USER_SEC}.{USER_USEC:03} \ + sys: {SYS_SEC}.{SYS_USEC:03} \ + max rss (kb): {MAXRSS}", + USER_SEC = rusage.ru_utime.tv_sec, + USER_USEC = rusage.ru_utime.tv_usec, + SYS_SEC = rusage.ru_stime.tv_sec, + SYS_USEC = rusage.ru_stime.tv_usec, + MAXRSS = maxrss + ); + + // The remaining rusage stats vary in platform support. So we treat + // uniformly zero values in each category as "not worth printing", since it + // either means no events of that type occurred, or that the platform + // does not support it. + + let minflt = rusage.ru_minflt; + let majflt = rusage.ru_majflt; + if minflt != 0 || majflt != 0 { + init_str.push_str(&format!(" page reclaims: {} page faults: {}", minflt, majflt)); + } + + let inblock = rusage.ru_inblock; + let oublock = rusage.ru_oublock; + if inblock != 0 || oublock != 0 { + init_str.push_str(&format!(" fs block inputs: {} fs block outputs: {}", inblock, oublock)); + } + + let nvcsw = rusage.ru_nvcsw; + let nivcsw = rusage.ru_nivcsw; + if nvcsw != 0 || nivcsw != 0 { + init_str.push_str(&format!( + " voluntary ctxt switches: {} involuntary ctxt switches: {}", + nvcsw, nivcsw + )); + } + + return Some(init_str); +} diff --git a/src/bootstrap/bin/rustdoc.rs b/src/bootstrap/bin/rustdoc.rs new file mode 100644 index 000000000..87c1d22e7 --- /dev/null +++ b/src/bootstrap/bin/rustdoc.rs @@ -0,0 +1,81 @@ +//! Shim which is passed to Cargo as "rustdoc" when running the bootstrap. +//! +//! See comments in `src/bootstrap/rustc.rs` for more information. + +use std::env; +use std::ffi::OsString; +use std::path::PathBuf; +use std::process::Command; + +include!("../dylib_util.rs"); + +fn main() { + let args = env::args_os().skip(1).collect::<Vec<_>>(); + let rustdoc = env::var_os("RUSTDOC_REAL").expect("RUSTDOC_REAL was not set"); + let libdir = env::var_os("RUSTDOC_LIBDIR").expect("RUSTDOC_LIBDIR was not set"); + let sysroot = env::var_os("RUSTC_SYSROOT").expect("RUSTC_SYSROOT was not set"); + + // Detect whether or not we're a build script depending on whether --target + // is passed (a bit janky...) + let target = args.windows(2).find(|w| &*w[0] == "--target").and_then(|w| w[1].to_str()); + + use std::str::FromStr; + + let verbose = match env::var("RUSTC_VERBOSE") { + Ok(s) => usize::from_str(&s).expect("RUSTC_VERBOSE should be an integer"), + Err(_) => 0, + }; + + let mut dylib_path = dylib_path(); + dylib_path.insert(0, PathBuf::from(libdir.clone())); + + let mut cmd = Command::new(rustdoc); + + if target.is_some() { + // The stage0 compiler has a special sysroot distinct from what we + // actually downloaded, so we just always pass the `--sysroot` option, + // unless one is already set. + if !args.iter().any(|arg| arg == "--sysroot") { + cmd.arg("--sysroot").arg(&sysroot); + } + } + + cmd.args(&args); + cmd.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); + + // Force all crates compiled by this compiler to (a) be unstable and (b) + // allow the `rustc_private` feature to link to other unstable crates + // also in the sysroot. + if env::var_os("RUSTC_FORCE_UNSTABLE").is_some() { + cmd.arg("-Z").arg("force-unstable-if-unmarked"); + } + if let Some(linker) = env::var_os("RUSTDOC_LINKER") { + let mut arg = OsString::from("-Clinker="); + arg.push(&linker); + cmd.arg(arg); + } + if env::var_os("RUSTDOC_FUSE_LD_LLD").is_some() { + cmd.arg("-Clink-arg=-fuse-ld=lld"); + if cfg!(windows) { + cmd.arg("-Clink-arg=-Wl,/threads:1"); + } else { + cmd.arg("-Clink-arg=-Wl,--threads=1"); + } + } + + if verbose > 1 { + eprintln!( + "rustdoc command: {:?}={:?} {:?}", + dylib_path_var(), + env::join_paths(&dylib_path).unwrap(), + cmd, + ); + eprintln!("sysroot: {:?}", sysroot); + eprintln!("libdir: {:?}", libdir); + } + + std::process::exit(match cmd.status() { + Ok(s) => s.code().unwrap_or(1), + Err(e) => panic!("\n\nfailed to run {:?}: {}\n\n", cmd, e), + }) +} diff --git a/src/bootstrap/bin/sccache-plus-cl.rs b/src/bootstrap/bin/sccache-plus-cl.rs new file mode 100644 index 000000000..554c2dd4d --- /dev/null +++ b/src/bootstrap/bin/sccache-plus-cl.rs @@ -0,0 +1,38 @@ +use std::env; +use std::process::{self, Command}; + +fn main() { + let target = env::var("SCCACHE_TARGET").unwrap(); + // Locate the actual compiler that we're invoking + env::set_var("CC", env::var_os("SCCACHE_CC").unwrap()); + env::set_var("CXX", env::var_os("SCCACHE_CXX").unwrap()); + let mut cfg = cc::Build::new(); + cfg.cargo_metadata(false) + .out_dir("/") + .target(&target) + .host(&target) + .opt_level(0) + .warnings(false) + .debug(false); + let compiler = cfg.get_compiler(); + + // Invoke sccache with said compiler + let sccache_path = env::var_os("SCCACHE_PATH").unwrap(); + let mut cmd = Command::new(&sccache_path); + cmd.arg(compiler.path()); + for &(ref k, ref v) in compiler.env() { + cmd.env(k, v); + } + for arg in env::args().skip(1) { + cmd.arg(arg); + } + + if let Ok(s) = env::var("SCCACHE_EXTRA_ARGS") { + for s in s.split_whitespace() { + cmd.arg(s); + } + } + + let status = cmd.status().expect("failed to spawn"); + process::exit(status.code().unwrap_or(2)) +} diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py new file mode 100644 index 000000000..03eec02a8 --- /dev/null +++ b/src/bootstrap/bootstrap.py @@ -0,0 +1,945 @@ +from __future__ import absolute_import, division, print_function +import argparse +import contextlib +import datetime +import distutils.version +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile + +from time import time, sleep + +def support_xz(): + try: + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + temp_path = temp_file.name + with tarfile.open(temp_path, "w:xz"): + pass + return True + except tarfile.CompressionError: + return False + +def get(base, url, path, checksums, verbose=False): + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + temp_path = temp_file.name + + try: + if url not in checksums: + raise RuntimeError(("src/stage0.json doesn't contain a checksum for {}. " + "Pre-built artifacts might not be available for this " + "target at this time, see https://doc.rust-lang.org/nightly" + "/rustc/platform-support.html for more information.") + .format(url)) + sha256 = checksums[url] + if os.path.exists(path): + if verify(path, sha256, False): + if verbose: + print("using already-download file", path) + return + else: + if verbose: + print("ignoring already-download file", + path, "due to failed verification") + os.unlink(path) + download(temp_path, "{}/{}".format(base, url), True, verbose) + if not verify(temp_path, sha256, verbose): + raise RuntimeError("failed verification") + if verbose: + print("moving {} to {}".format(temp_path, path)) + shutil.move(temp_path, path) + finally: + if os.path.isfile(temp_path): + if verbose: + print("removing", temp_path) + os.unlink(temp_path) + + +def download(path, url, probably_big, verbose): + for _ in range(0, 4): + try: + _download(path, url, probably_big, verbose, True) + return + except RuntimeError: + print("\nspurious failure, trying again") + _download(path, url, probably_big, verbose, False) + + +def _download(path, url, probably_big, verbose, exception): + # Try to use curl (potentially available on win32 + # https://devblogs.microsoft.com/commandline/tar-and-curl-come-to-windows/) + # If an error occurs: + # - If we are on win32 fallback to powershell + # - Otherwise raise the error if appropriate + if probably_big or verbose: + print("downloading {}".format(url)) + + platform_is_win32 = sys.platform == 'win32' + try: + if probably_big or verbose: + option = "-#" + else: + option = "-s" + # If curl is not present on Win32, we shoud not sys.exit + # but raise `CalledProcessError` or `OSError` instead + require(["curl", "--version"], exception=platform_is_win32) + run(["curl", option, + "-L", # Follow redirect. + "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds + "--connect-timeout", "30", # timeout if cannot connect within 30 seconds + "--retry", "3", "-Sf", "-o", path, url], + verbose=verbose, + exception=True, # Will raise RuntimeError on failure + ) + except (subprocess.CalledProcessError, OSError, RuntimeError): + # see http://serverfault.com/questions/301128/how-to-download + if platform_is_win32: + run(["PowerShell.exe", "/nologo", "-Command", + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", + "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)], + verbose=verbose, + exception=exception) + # Check if the RuntimeError raised by run(curl) should be silenced + elif verbose or exception: + raise + + +def verify(path, expected, verbose): + """Check if the sha256 sum of the given path is valid""" + if verbose: + print("verifying", path) + with open(path, "rb") as source: + found = hashlib.sha256(source.read()).hexdigest() + verified = found == expected + if not verified: + print("invalid checksum:\n" + " found: {}\n" + " expected: {}".format(found, expected)) + return verified + + +def unpack(tarball, tarball_suffix, dst, verbose=False, match=None): + """Unpack the given tarball file""" + print("extracting", tarball) + fname = os.path.basename(tarball).replace(tarball_suffix, "") + with contextlib.closing(tarfile.open(tarball)) as tar: + for member in tar.getnames(): + if "/" not in member: + continue + name = member.replace(fname + "/", "", 1) + if match is not None and not name.startswith(match): + continue + name = name[len(match) + 1:] + + dst_path = os.path.join(dst, name) + if verbose: + print(" extracting", member) + tar.extract(member, dst) + src_path = os.path.join(dst, member) + if os.path.isdir(src_path) and os.path.exists(dst_path): + continue + shutil.move(src_path, dst_path) + shutil.rmtree(os.path.join(dst, fname)) + + +def run(args, verbose=False, exception=False, is_bootstrap=False, **kwargs): + """Run a child program in a new process""" + if verbose: + print("running: " + ' '.join(args)) + sys.stdout.flush() + # Ensure that the .exe is used on Windows just in case a Linux ELF has been + # compiled in the same directory. + if os.name == 'nt' and not args[0].endswith('.exe'): + args[0] += '.exe' + # Use Popen here instead of call() as it apparently allows powershell on + # Windows to not lock up waiting for input presumably. + ret = subprocess.Popen(args, **kwargs) + code = ret.wait() + if code != 0: + err = "failed to run: " + ' '.join(args) + if verbose or exception: + raise RuntimeError(err) + # For most failures, we definitely do want to print this error, or the user will have no + # idea what went wrong. But when we've successfully built bootstrap and it failed, it will + # have already printed an error above, so there's no need to print the exact command we're + # running. + if is_bootstrap: + sys.exit(1) + else: + sys.exit(err) + + +def require(cmd, exit=True, exception=False): + '''Run a command, returning its output. + On error, + If `exception` is `True`, raise the error + Otherwise If `exit` is `True`, exit the process + Else return None.''' + try: + return subprocess.check_output(cmd).strip() + except (subprocess.CalledProcessError, OSError) as exc: + if exception: + raise + elif exit: + print("error: unable to run `{}`: {}".format(' '.join(cmd), exc)) + print("Please make sure it's installed and in the path.") + sys.exit(1) + return None + + + +def format_build_time(duration): + """Return a nicer format for build time + + >>> format_build_time('300') + '0:05:00' + """ + return str(datetime.timedelta(seconds=int(duration))) + + +def default_build_triple(verbose): + """Build triple as in LLVM""" + # If the user already has a host build triple with an existing `rustc` + # install, use their preference. This fixes most issues with Windows builds + # being detected as GNU instead of MSVC. + default_encoding = sys.getdefaultencoding() + try: + version = subprocess.check_output(["rustc", "--version", "--verbose"], + stderr=subprocess.DEVNULL) + version = version.decode(default_encoding) + host = next(x for x in version.split('\n') if x.startswith("host: ")) + triple = host.split("host: ")[1] + if verbose: + print("detected default triple {} from pre-installed rustc".format(triple)) + return triple + except Exception as e: + if verbose: + print("pre-installed rustc not detected: {}".format(e)) + print("falling back to auto-detect") + + required = sys.platform != 'win32' + ostype = require(["uname", "-s"], exit=required) + cputype = require(['uname', '-m'], exit=required) + + # If we do not have `uname`, assume Windows. + if ostype is None or cputype is None: + return 'x86_64-pc-windows-msvc' + + ostype = ostype.decode(default_encoding) + cputype = cputype.decode(default_encoding) + + # The goal here is to come up with the same triple as LLVM would, + # at least for the subset of platforms we're willing to target. + ostype_mapper = { + 'Darwin': 'apple-darwin', + 'DragonFly': 'unknown-dragonfly', + 'FreeBSD': 'unknown-freebsd', + 'Haiku': 'unknown-haiku', + 'NetBSD': 'unknown-netbsd', + 'OpenBSD': 'unknown-openbsd' + } + + # Consider the direct transformation first and then the special cases + if ostype in ostype_mapper: + ostype = ostype_mapper[ostype] + elif ostype == 'Linux': + os_from_sp = subprocess.check_output( + ['uname', '-o']).strip().decode(default_encoding) + if os_from_sp == 'Android': + ostype = 'linux-android' + else: + ostype = 'unknown-linux-gnu' + elif ostype == 'SunOS': + ostype = 'pc-solaris' + # On Solaris, uname -m will return a machine classification instead + # of a cpu type, so uname -p is recommended instead. However, the + # output from that option is too generic for our purposes (it will + # always emit 'i386' on x86/amd64 systems). As such, isainfo -k + # must be used instead. + cputype = require(['isainfo', '-k']).decode(default_encoding) + # sparc cpus have sun as a target vendor + if 'sparc' in cputype: + ostype = 'sun-solaris' + elif ostype.startswith('MINGW'): + # msys' `uname` does not print gcc configuration, but prints msys + # configuration. so we cannot believe `uname -m`: + # msys1 is always i686 and msys2 is always x86_64. + # instead, msys defines $MSYSTEM which is MINGW32 on i686 and + # MINGW64 on x86_64. + ostype = 'pc-windows-gnu' + cputype = 'i686' + if os.environ.get('MSYSTEM') == 'MINGW64': + cputype = 'x86_64' + elif ostype.startswith('MSYS'): + ostype = 'pc-windows-gnu' + elif ostype.startswith('CYGWIN_NT'): + cputype = 'i686' + if ostype.endswith('WOW64'): + cputype = 'x86_64' + ostype = 'pc-windows-gnu' + elif sys.platform == 'win32': + # Some Windows platforms might have a `uname` command that returns a + # non-standard string (e.g. gnuwin32 tools returns `windows32`). In + # these cases, fall back to using sys.platform. + return 'x86_64-pc-windows-msvc' + else: + err = "unknown OS type: {}".format(ostype) + sys.exit(err) + + if cputype in ['powerpc', 'riscv'] and ostype == 'unknown-freebsd': + cputype = subprocess.check_output( + ['uname', '-p']).strip().decode(default_encoding) + cputype_mapper = { + 'BePC': 'i686', + 'aarch64': 'aarch64', + 'amd64': 'x86_64', + 'arm64': 'aarch64', + 'i386': 'i686', + 'i486': 'i686', + 'i686': 'i686', + 'i786': 'i686', + 'm68k': 'm68k', + 'powerpc': 'powerpc', + 'powerpc64': 'powerpc64', + 'powerpc64le': 'powerpc64le', + 'ppc': 'powerpc', + 'ppc64': 'powerpc64', + 'ppc64le': 'powerpc64le', + 'riscv64': 'riscv64gc', + 's390x': 's390x', + 'x64': 'x86_64', + 'x86': 'i686', + 'x86-64': 'x86_64', + 'x86_64': 'x86_64' + } + + # Consider the direct transformation first and then the special cases + if cputype in cputype_mapper: + cputype = cputype_mapper[cputype] + elif cputype in {'xscale', 'arm'}: + cputype = 'arm' + if ostype == 'linux-android': + ostype = 'linux-androideabi' + elif ostype == 'unknown-freebsd': + cputype = subprocess.check_output( + ['uname', '-p']).strip().decode(default_encoding) + ostype = 'unknown-freebsd' + elif cputype == 'armv6l': + cputype = 'arm' + if ostype == 'linux-android': + ostype = 'linux-androideabi' + else: + ostype += 'eabihf' + elif cputype in {'armv7l', 'armv8l'}: + cputype = 'armv7' + if ostype == 'linux-android': + ostype = 'linux-androideabi' + else: + ostype += 'eabihf' + elif cputype == 'mips': + if sys.byteorder == 'big': + cputype = 'mips' + elif sys.byteorder == 'little': + cputype = 'mipsel' + else: + raise ValueError("unknown byteorder: {}".format(sys.byteorder)) + elif cputype == 'mips64': + if sys.byteorder == 'big': + cputype = 'mips64' + elif sys.byteorder == 'little': + cputype = 'mips64el' + else: + raise ValueError('unknown byteorder: {}'.format(sys.byteorder)) + # only the n64 ABI is supported, indicate it + ostype += 'abi64' + elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64': + pass + else: + err = "unknown cpu type: {}".format(cputype) + sys.exit(err) + + return "{}-{}".format(cputype, ostype) + + +@contextlib.contextmanager +def output(filepath): + tmp = filepath + '.tmp' + with open(tmp, 'w') as f: + yield f + try: + if os.path.exists(filepath): + os.remove(filepath) # PermissionError/OSError on Win32 if in use + except OSError: + shutil.copy2(tmp, filepath) + os.remove(tmp) + return + os.rename(tmp, filepath) + + +class Stage0Toolchain: + def __init__(self, stage0_payload): + self.date = stage0_payload["date"] + self.version = stage0_payload["version"] + + def channel(self): + return self.version + "-" + self.date + + +class RustBuild(object): + """Provide all the methods required to build Rust""" + def __init__(self): + self.checksums_sha256 = {} + self.stage0_compiler = None + self._download_url = '' + self.build = '' + self.build_dir = '' + self.clean = False + self.config_toml = '' + self.rust_root = '' + self.use_locked_deps = '' + self.use_vendored_sources = '' + self.verbose = False + self.git_version = None + self.nix_deps_dir = None + + def download_toolchain(self): + """Fetch the build system for Rust, written in Rust + + This method will build a cache directory, then it will fetch the + tarball which has the stage0 compiler used to then bootstrap the Rust + compiler itself. + + Each downloaded tarball is extracted, after that, the script + will move all the content to the right place. + """ + rustc_channel = self.stage0_compiler.version + bin_root = self.bin_root() + + key = self.stage0_compiler.date + if self.rustc().startswith(bin_root) and \ + (not os.path.exists(self.rustc()) or + self.program_out_of_date(self.rustc_stamp(), key)): + if os.path.exists(bin_root): + shutil.rmtree(bin_root) + tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' + filename = "rust-std-{}-{}{}".format( + rustc_channel, self.build, tarball_suffix) + pattern = "rust-std-{}".format(self.build) + self._download_component_helper(filename, pattern, tarball_suffix) + filename = "rustc-{}-{}{}".format(rustc_channel, self.build, + tarball_suffix) + self._download_component_helper(filename, "rustc", tarball_suffix) + filename = "cargo-{}-{}{}".format(rustc_channel, self.build, + tarball_suffix) + self._download_component_helper(filename, "cargo", tarball_suffix) + self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root)) + + self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root)) + self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root)) + lib_dir = "{}/lib".format(bin_root) + for lib in os.listdir(lib_dir): + if lib.endswith(".so"): + self.fix_bin_or_dylib(os.path.join(lib_dir, lib)) + with output(self.rustc_stamp()) as rust_stamp: + rust_stamp.write(key) + + def _download_component_helper( + self, filename, pattern, tarball_suffix, + ): + key = self.stage0_compiler.date + cache_dst = os.path.join(self.build_dir, "cache") + rustc_cache = os.path.join(cache_dst, key) + if not os.path.exists(rustc_cache): + os.makedirs(rustc_cache) + + base = self._download_url + url = "dist/{}".format(key) + tarball = os.path.join(rustc_cache, filename) + if not os.path.exists(tarball): + get( + base, + "{}/{}".format(url, filename), + tarball, + self.checksums_sha256, + verbose=self.verbose, + ) + unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose) + + def fix_bin_or_dylib(self, fname): + """Modifies the interpreter section of 'fname' to fix the dynamic linker, + or the RPATH section, to fix the dynamic library search path + + This method is only required on NixOS and uses the PatchELF utility to + change the interpreter/RPATH of ELF executables. + + Please see https://nixos.org/patchelf.html for more information + """ + default_encoding = sys.getdefaultencoding() + try: + ostype = subprocess.check_output( + ['uname', '-s']).strip().decode(default_encoding) + except subprocess.CalledProcessError: + return + except OSError as reason: + if getattr(reason, 'winerror', None) is not None: + return + raise reason + + if ostype != "Linux": + return + + # If the user has asked binaries to be patched for Nix, then + # don't check for NixOS or `/lib`, just continue to the patching. + if self.get_toml('patch-binaries-for-nix', 'build') != 'true': + # Use `/etc/os-release` instead of `/etc/NIXOS`. + # The latter one does not exist on NixOS when using tmpfs as root. + try: + with open("/etc/os-release", "r") as f: + if not any(l.strip() in ["ID=nixos", "ID='nixos'", 'ID="nixos"'] for l in f): + return + except FileNotFoundError: + return + if os.path.exists("/lib"): + return + + # At this point we're pretty sure the user is running NixOS or + # using Nix + nix_os_msg = "info: you seem to be using Nix. Attempting to patch" + print(nix_os_msg, fname) + + # Only build `.nix-deps` once. + nix_deps_dir = self.nix_deps_dir + if not nix_deps_dir: + # Run `nix-build` to "build" each dependency (which will likely reuse + # the existing `/nix/store` copy, or at most download a pre-built copy). + # + # Importantly, we create a gc-root called `.nix-deps` in the `build/` + # directory, but still reference the actual `/nix/store` path in the rpath + # as it makes it significantly more robust against changes to the location of + # the `.nix-deps` location. + # + # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`). + # zlib: Needed as a system dependency of `libLLVM-*.so`. + # patchelf: Needed for patching ELF binaries (see doc comment above). + nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps") + nix_expr = ''' + with (import <nixpkgs> {}); + symlinkJoin { + name = "rust-stage0-dependencies"; + paths = [ + zlib + patchelf + stdenv.cc.bintools + ]; + } + ''' + try: + subprocess.check_output([ + "nix-build", "-E", nix_expr, "-o", nix_deps_dir, + ]) + except subprocess.CalledProcessError as reason: + print("warning: failed to call nix-build:", reason) + return + self.nix_deps_dir = nix_deps_dir + + patchelf = "{}/bin/patchelf".format(nix_deps_dir) + rpath_entries = [ + # Relative default, all binary and dynamic libraries we ship + # appear to have this (even when `../lib` is redundant). + "$ORIGIN/../lib", + os.path.join(os.path.realpath(nix_deps_dir), "lib") + ] + patchelf_args = ["--set-rpath", ":".join(rpath_entries)] + if not fname.endswith(".so"): + # Finally, set the corret .interp for binaries + with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker: + patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()] + + try: + subprocess.check_output([patchelf] + patchelf_args + [fname]) + except subprocess.CalledProcessError as reason: + print("warning: failed to call patchelf:", reason) + return + + def rustc_stamp(self): + """Return the path for .rustc-stamp at the given stage + + >>> rb = RustBuild() + >>> rb.build_dir = "build" + >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp") + True + """ + return os.path.join(self.bin_root(), '.rustc-stamp') + + def program_out_of_date(self, stamp_path, key): + """Check if the given program stamp is out of date""" + if not os.path.exists(stamp_path) or self.clean: + return True + with open(stamp_path, 'r') as stamp: + return key != stamp.read() + + def bin_root(self): + """Return the binary root directory for the given stage + + >>> rb = RustBuild() + >>> rb.build_dir = "build" + >>> rb.bin_root() == os.path.join("build", "stage0") + True + + When the 'build' property is given should be a nested directory: + + >>> rb.build = "devel" + >>> rb.bin_root() == os.path.join("build", "devel", "stage0") + True + """ + subdir = "stage0" + return os.path.join(self.build_dir, self.build, subdir) + + def get_toml(self, key, section=None): + """Returns the value of the given key in config.toml, otherwise returns None + + >>> rb = RustBuild() + >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"' + >>> rb.get_toml("key2") + 'value2' + + If the key does not exist, the result is None: + + >>> rb.get_toml("key3") is None + True + + Optionally also matches the section the key appears in + + >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"' + >>> rb.get_toml('key', 'a') + 'value1' + >>> rb.get_toml('key', 'b') + 'value2' + >>> rb.get_toml('key', 'c') is None + True + + >>> rb.config_toml = 'key1 = true' + >>> rb.get_toml("key1") + 'true' + """ + + cur_section = None + for line in self.config_toml.splitlines(): + section_match = re.match(r'^\s*\[(.*)\]\s*$', line) + if section_match is not None: + cur_section = section_match.group(1) + + match = re.match(r'^{}\s*=(.*)$'.format(key), line) + if match is not None: + value = match.group(1) + if section is None or section == cur_section: + return self.get_string(value) or value.strip() + return None + + def cargo(self): + """Return config path for cargo""" + return self.program_config('cargo') + + def rustc(self): + """Return config path for rustc""" + return self.program_config('rustc') + + def program_config(self, program): + """Return config path for the given program at the given stage + + >>> rb = RustBuild() + >>> rb.config_toml = 'rustc = "rustc"\\n' + >>> rb.program_config('rustc') + 'rustc' + >>> rb.config_toml = '' + >>> cargo_path = rb.program_config('cargo') + >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(), + ... "bin", "cargo") + True + """ + config = self.get_toml(program) + if config: + return os.path.expanduser(config) + return os.path.join(self.bin_root(), "bin", "{}{}".format( + program, self.exe_suffix())) + + @staticmethod + def get_string(line): + """Return the value between double quotes + + >>> RustBuild.get_string(' "devel" ') + 'devel' + >>> RustBuild.get_string(" 'devel' ") + 'devel' + >>> RustBuild.get_string('devel') is None + True + >>> RustBuild.get_string(' "devel ') + '' + """ + start = line.find('"') + if start != -1: + end = start + 1 + line[start + 1:].find('"') + return line[start + 1:end] + start = line.find('\'') + if start != -1: + end = start + 1 + line[start + 1:].find('\'') + return line[start + 1:end] + return None + + @staticmethod + def exe_suffix(): + """Return a suffix for executables""" + if sys.platform == 'win32': + return '.exe' + return '' + + def bootstrap_binary(self): + """Return the path of the bootstrap binary + + >>> rb = RustBuild() + >>> rb.build_dir = "build" + >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap", + ... "debug", "bootstrap") + True + """ + return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap") + + def build_bootstrap(self, color): + """Build bootstrap""" + print("Building rustbuild") + build_dir = os.path.join(self.build_dir, "bootstrap") + if self.clean and os.path.exists(build_dir): + shutil.rmtree(build_dir) + env = os.environ.copy() + # `CARGO_BUILD_TARGET` breaks bootstrap build. + # See also: <https://github.com/rust-lang/rust/issues/70208>. + if "CARGO_BUILD_TARGET" in env: + del env["CARGO_BUILD_TARGET"] + env["CARGO_TARGET_DIR"] = build_dir + env["RUSTC"] = self.rustc() + env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ + (os.pathsep + env["LD_LIBRARY_PATH"]) \ + if "LD_LIBRARY_PATH" in env else "" + env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ + (os.pathsep + env["DYLD_LIBRARY_PATH"]) \ + if "DYLD_LIBRARY_PATH" in env else "" + env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ + (os.pathsep + env["LIBRARY_PATH"]) \ + if "LIBRARY_PATH" in env else "" + + # preserve existing RUSTFLAGS + env.setdefault("RUSTFLAGS", "") + build_section = "target.{}".format(self.build) + target_features = [] + if self.get_toml("crt-static", build_section) == "true": + target_features += ["+crt-static"] + elif self.get_toml("crt-static", build_section) == "false": + target_features += ["-crt-static"] + if target_features: + env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features)) + target_linker = self.get_toml("linker", build_section) + if target_linker is not None: + env["RUSTFLAGS"] += " -C linker=" + target_linker + env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes" + env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros" + if self.get_toml("deny-warnings", "rust") != "false": + env["RUSTFLAGS"] += " -Dwarnings" + + env["PATH"] = os.path.join(self.bin_root(), "bin") + \ + os.pathsep + env["PATH"] + if not os.path.isfile(self.cargo()): + raise Exception("no cargo executable found at `{}`".format( + self.cargo())) + args = [self.cargo(), "build", "--manifest-path", + os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")] + for _ in range(0, self.verbose): + args.append("--verbose") + if self.use_locked_deps: + args.append("--locked") + if self.use_vendored_sources: + args.append("--frozen") + if self.get_toml("metrics", "build"): + args.append("--features") + args.append("build-metrics") + if color == "always": + args.append("--color=always") + elif color == "never": + args.append("--color=never") + + run(args, env=env, verbose=self.verbose) + + def build_triple(self): + """Build triple as in LLVM + + Note that `default_build_triple` is moderately expensive, + so use `self.build` where possible. + """ + config = self.get_toml('build') + if config: + return config + return default_build_triple(self.verbose) + + def set_dist_environment(self, url): + """Set download URL for normal environment""" + if 'RUSTUP_DIST_SERVER' in os.environ: + self._download_url = os.environ['RUSTUP_DIST_SERVER'] + else: + self._download_url = url + + def check_vendored_status(self): + """Check that vendoring is configured properly""" + # keep this consistent with the equivalent check in rustbuild: + # https://github.com/rust-lang/rust/blob/a8a33cf27166d3eabaffc58ed3799e054af3b0c6/src/bootstrap/lib.rs#L399-L405 + if 'SUDO_USER' in os.environ and not self.use_vendored_sources: + if os.getuid() == 0: + self.use_vendored_sources = True + print('info: looks like you\'re trying to run this command as root') + print(' and so in order to preserve your $HOME this will now') + print(' use vendored sources by default.') + + cargo_dir = os.path.join(self.rust_root, '.cargo') + if self.use_vendored_sources: + vendor_dir = os.path.join(self.rust_root, 'vendor') + if not os.path.exists(vendor_dir): + sync_dirs = "--sync ./src/tools/rust-analyzer/Cargo.toml " \ + "--sync ./compiler/rustc_codegen_cranelift/Cargo.toml " \ + "--sync ./src/bootstrap/Cargo.toml " + print('error: vendoring required, but vendor directory does not exist.') + print(' Run `cargo vendor {}` to initialize the ' + 'vendor directory.'.format(sync_dirs)) + print('Alternatively, use the pre-vendored `rustc-src` dist component.') + raise Exception("{} not found".format(vendor_dir)) + + if not os.path.exists(cargo_dir): + print('error: vendoring required, but .cargo/config does not exist.') + raise Exception("{} not found".format(cargo_dir)) + else: + if os.path.exists(cargo_dir): + shutil.rmtree(cargo_dir) + +def bootstrap(help_triggered): + """Configure, fetch, build and run the initial bootstrap""" + + # If the user is asking for help, let them know that the whole download-and-build + # process has to happen before anything is printed out. + if help_triggered: + print("info: Downloading and building bootstrap before processing --help") + print(" command. See src/bootstrap/README.md for help with common") + print(" commands.") + + parser = argparse.ArgumentParser(description='Build rust') + parser.add_argument('--config') + parser.add_argument('--build-dir') + parser.add_argument('--build') + parser.add_argument('--color', choices=['always', 'never', 'auto']) + parser.add_argument('--clean', action='store_true') + parser.add_argument('-v', '--verbose', action='count', default=0) + + args = [a for a in sys.argv if a != '-h' and a != '--help'] + args, _ = parser.parse_known_args(args) + + # Configure initial bootstrap + build = RustBuild() + build.rust_root = os.path.abspath(os.path.join(__file__, '../../..')) + build.verbose = args.verbose + build.clean = args.clean + + # Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, + # then `config.toml` in the root directory. + toml_path = args.config or os.getenv('RUST_BOOTSTRAP_CONFIG') + using_default_path = toml_path is None + if using_default_path: + toml_path = 'config.toml' + if not os.path.exists(toml_path): + toml_path = os.path.join(build.rust_root, toml_path) + + # Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path, + # but not if `config.toml` hasn't been created. + if not using_default_path or os.path.exists(toml_path): + with open(toml_path) as config: + build.config_toml = config.read() + + profile = build.get_toml('profile') + if profile is not None: + include_file = 'config.{}.toml'.format(profile) + include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults') + include_path = os.path.join(include_dir, include_file) + # HACK: This works because `build.get_toml()` returns the first match it finds for a + # specific key, so appending our defaults at the end allows the user to override them + with open(include_path) as included_toml: + build.config_toml += os.linesep + included_toml.read() + + config_verbose = build.get_toml('verbose', 'build') + if config_verbose is not None: + build.verbose = max(build.verbose, int(config_verbose)) + + build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true' + + build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true' + + build.check_vendored_status() + + build_dir = args.build_dir or build.get_toml('build-dir', 'build') or 'build' + build.build_dir = os.path.abspath(build_dir) + + with open(os.path.join(build.rust_root, "src", "stage0.json")) as f: + data = json.load(f) + build.checksums_sha256 = data["checksums_sha256"] + build.stage0_compiler = Stage0Toolchain(data["compiler"]) + + build.set_dist_environment(data["config"]["dist_server"]) + + build.build = args.build or build.build_triple() + + if not os.path.exists(build.build_dir): + os.makedirs(build.build_dir) + + # Fetch/build the bootstrap + build.download_toolchain() + sys.stdout.flush() + build.build_bootstrap(args.color) + sys.stdout.flush() + + # Run the bootstrap + args = [build.bootstrap_binary()] + args.extend(sys.argv[1:]) + env = os.environ.copy() + env["BOOTSTRAP_PARENT_ID"] = str(os.getpid()) + env["BOOTSTRAP_PYTHON"] = sys.executable + run(args, env=env, verbose=build.verbose, is_bootstrap=True) + + +def main(): + """Entry point for the bootstrap process""" + start_time = time() + + # x.py help <cmd> ... + if len(sys.argv) > 1 and sys.argv[1] == 'help': + sys.argv = [sys.argv[0], '-h'] + sys.argv[2:] + + help_triggered = ( + '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1) + try: + bootstrap(help_triggered) + if not help_triggered: + print("Build completed successfully in {}".format( + format_build_time(time() - start_time))) + except (SystemExit, KeyboardInterrupt) as error: + if hasattr(error, 'code') and isinstance(error.code, int): + exit_code = error.code + else: + exit_code = 1 + print(error) + if not help_triggered: + print("Build completed unsuccessfully in {}".format( + format_build_time(time() - start_time))) + sys.exit(exit_code) + + +if __name__ == '__main__': + main() diff --git a/src/bootstrap/bootstrap_test.py b/src/bootstrap/bootstrap_test.py new file mode 100644 index 000000000..06ca3ce21 --- /dev/null +++ b/src/bootstrap/bootstrap_test.py @@ -0,0 +1,87 @@ +"""Bootstrap tests""" + +from __future__ import absolute_import, division, print_function +import os +import doctest +import unittest +import tempfile +import hashlib +import sys + +from shutil import rmtree + +import bootstrap + + +class VerifyTestCase(unittest.TestCase): + """Test Case for verify""" + def setUp(self): + self.container = tempfile.mkdtemp() + self.src = os.path.join(self.container, "src.txt") + self.bad_src = os.path.join(self.container, "bad.txt") + content = "Hello world" + + self.expected = hashlib.sha256(content.encode("utf-8")).hexdigest() + + with open(self.src, "w") as src: + src.write(content) + with open(self.bad_src, "w") as bad: + bad.write("Hello!") + + def tearDown(self): + rmtree(self.container) + + def test_valid_file(self): + """Check if the sha256 sum of the given file is valid""" + self.assertTrue(bootstrap.verify(self.src, self.expected, False)) + + def test_invalid_file(self): + """Should verify that the file is invalid""" + self.assertFalse(bootstrap.verify(self.bad_src, self.expected, False)) + + +class ProgramOutOfDate(unittest.TestCase): + """Test if a program is out of date""" + def setUp(self): + self.container = tempfile.mkdtemp() + os.mkdir(os.path.join(self.container, "stage0")) + self.build = bootstrap.RustBuild() + self.build.date = "2017-06-15" + self.build.build_dir = self.container + self.rustc_stamp_path = os.path.join(self.container, "stage0", + ".rustc-stamp") + self.key = self.build.date + str(None) + + def tearDown(self): + rmtree(self.container) + + def test_stamp_path_does_not_exist(self): + """Return True when the stamp file does not exist""" + if os.path.exists(self.rustc_stamp_path): + os.unlink(self.rustc_stamp_path) + self.assertTrue(self.build.program_out_of_date(self.rustc_stamp_path, self.key)) + + def test_dates_are_different(self): + """Return True when the dates are different""" + with open(self.rustc_stamp_path, "w") as rustc_stamp: + rustc_stamp.write("2017-06-14None") + self.assertTrue(self.build.program_out_of_date(self.rustc_stamp_path, self.key)) + + def test_same_dates(self): + """Return False both dates match""" + with open(self.rustc_stamp_path, "w") as rustc_stamp: + rustc_stamp.write("2017-06-15None") + self.assertFalse(self.build.program_out_of_date(self.rustc_stamp_path, self.key)) + + +if __name__ == '__main__': + SUITE = unittest.TestSuite() + TEST_LOADER = unittest.TestLoader() + SUITE.addTest(doctest.DocTestSuite(bootstrap)) + SUITE.addTests([ + TEST_LOADER.loadTestsFromTestCase(VerifyTestCase), + TEST_LOADER.loadTestsFromTestCase(ProgramOutOfDate)]) + + RUNNER = unittest.TextTestRunner(stream=sys.stdout, verbosity=2) + result = RUNNER.run(SUITE) + sys.exit(0 if result.wasSuccessful() else 1) diff --git a/src/bootstrap/build.rs b/src/bootstrap/build.rs new file mode 100644 index 000000000..ab34d5c1e --- /dev/null +++ b/src/bootstrap/build.rs @@ -0,0 +1,43 @@ +use env::consts::{EXE_EXTENSION, EXE_SUFFIX}; +use std::env; +use std::ffi::OsString; +use std::path::PathBuf; + +/// Given an executable called `name`, return the filename for the +/// executable for a particular target. +pub fn exe(name: &PathBuf) -> PathBuf { + if EXE_EXTENSION != "" && name.extension() != Some(EXE_EXTENSION.as_ref()) { + let mut name: OsString = name.clone().into(); + name.push(EXE_SUFFIX); + name.into() + } else { + name.clone() + } +} + +fn main() { + let host = env::var("HOST").unwrap(); + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-env-changed=RUSTC"); + println!("cargo:rustc-env=BUILD_TRIPLE={}", host); + + // This may not be a canonicalized path. + let mut rustc = PathBuf::from(env::var_os("RUSTC").unwrap()); + + if rustc.is_relative() { + println!("cargo:rerun-if-env-changed=PATH"); + for dir in env::split_paths(&env::var_os("PATH").unwrap_or_default()) { + let absolute = dir.join(&exe(&rustc)); + if absolute.exists() { + rustc = absolute; + break; + } + } + } + assert!(rustc.is_absolute()); + + // FIXME: if the path is not utf-8, this is going to break. Unfortunately + // Cargo doesn't have a way for us to specify non-utf-8 paths easily, so + // we'll need to invent some encoding scheme if this becomes a problem. + println!("cargo:rustc-env=RUSTC={}", rustc.to_str().unwrap()); +} diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs new file mode 100644 index 000000000..0ab4824ac --- /dev/null +++ b/src/bootstrap/builder.rs @@ -0,0 +1,2312 @@ +use std::any::{type_name, Any}; +use std::cell::{Cell, RefCell}; +use std::collections::BTreeSet; +use std::env; +use std::ffi::{OsStr, OsString}; +use std::fmt::{Debug, Write}; +use std::fs::{self, File}; +use std::hash::Hash; +use std::io::{BufRead, BufReader, ErrorKind}; +use std::ops::Deref; +use std::path::{Component, Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +use crate::cache::{Cache, Interned, INTERNER}; +use crate::config::{SplitDebuginfo, TargetSelection}; +use crate::dist; +use crate::doc; +use crate::flags::{Color, Subcommand}; +use crate::install; +use crate::native; +use crate::run; +use crate::test; +use crate::tool::{self, SourceType}; +use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir, output, t}; +use crate::EXTRA_CHECK_CFGS; +use crate::{check, Config}; +use crate::{compile, Crate}; +use crate::{Build, CLang, DocTests, GitRepo, Mode}; + +pub use crate::Compiler; +// FIXME: replace with std::lazy after it gets stabilized and reaches beta +use once_cell::sync::{Lazy, OnceCell}; +use xz2::bufread::XzDecoder; + +pub struct Builder<'a> { + pub build: &'a Build, + pub top_stage: u32, + pub kind: Kind, + cache: Cache, + stack: RefCell<Vec<Box<dyn Any>>>, + time_spent_on_dependencies: Cell<Duration>, + pub paths: Vec<PathBuf>, +} + +impl<'a> Deref for Builder<'a> { + type Target = Build; + + fn deref(&self) -> &Self::Target { + self.build + } +} + +pub trait Step: 'static + Clone + Debug + PartialEq + Eq + Hash { + /// `PathBuf` when directories are created or to return a `Compiler` once + /// it's been assembled. + type Output: Clone; + + /// Whether this step is run by default as part of its respective phase. + /// `true` here can still be overwritten by `should_run` calling `default_condition`. + const DEFAULT: bool = false; + + /// If true, then this rule should be skipped if --target was specified, but --host was not + const ONLY_HOSTS: bool = false; + + /// Primary function to execute this rule. Can call `builder.ensure()` + /// with other steps to run those. + fn run(self, builder: &Builder<'_>) -> Self::Output; + + /// When bootstrap is passed a set of paths, this controls whether this rule + /// will execute. However, it does not get called in a "default" context + /// when we are not passed any paths; in that case, `make_run` is called + /// directly. + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_>; + + /// Builds up a "root" rule, either as a default rule or from a path passed + /// to us. + /// + /// When path is `None`, we are executing in a context where no paths were + /// passed. When `./x.py build` is run, for example, this rule could get + /// called if it is in the correct list below with a path of `None`. + fn make_run(_run: RunConfig<'_>) { + // It is reasonable to not have an implementation of make_run for rules + // who do not want to get called from the root context. This means that + // they are likely dependencies (e.g., sysroot creation) or similar, and + // as such calling them from ./x.py isn't logical. + unimplemented!() + } +} + +pub struct RunConfig<'a> { + pub builder: &'a Builder<'a>, + pub target: TargetSelection, + pub paths: Vec<PathSet>, +} + +impl RunConfig<'_> { + pub fn build_triple(&self) -> TargetSelection { + self.builder.build.build + } +} + +struct StepDescription { + default: bool, + only_hosts: bool, + should_run: fn(ShouldRun<'_>) -> ShouldRun<'_>, + make_run: fn(RunConfig<'_>), + name: &'static str, + kind: Kind, +} + +#[derive(Clone, PartialOrd, Ord, PartialEq, Eq)] +pub struct TaskPath { + pub path: PathBuf, + pub kind: Option<Kind>, +} + +impl TaskPath { + pub fn parse(path: impl Into<PathBuf>) -> TaskPath { + let mut kind = None; + let mut path = path.into(); + + let mut components = path.components(); + if let Some(Component::Normal(os_str)) = components.next() { + if let Some(str) = os_str.to_str() { + if let Some((found_kind, found_prefix)) = str.split_once("::") { + if found_kind.is_empty() { + panic!("empty kind in task path {}", path.display()); + } + kind = Kind::parse(found_kind); + assert!(kind.is_some()); + path = Path::new(found_prefix).join(components.as_path()); + } + } + } + + TaskPath { path, kind } + } +} + +impl Debug for TaskPath { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some(kind) = &self.kind { + write!(f, "{}::", kind.as_str())?; + } + write!(f, "{}", self.path.display()) + } +} + +/// Collection of paths used to match a task rule. +#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)] +pub enum PathSet { + /// A collection of individual paths or aliases. + /// + /// These are generally matched as a path suffix. For example, a + /// command-line value of `std` will match if `library/std` is in the + /// set. + /// + /// NOTE: the paths within a set should always be aliases of one another. + /// For example, `src/librustdoc` and `src/tools/rustdoc` should be in the same set, + /// but `library/core` and `library/std` generally should not, unless there's no way (for that Step) + /// to build them separately. + Set(BTreeSet<TaskPath>), + /// A "suite" of paths. + /// + /// These can match as a path suffix (like `Set`), or as a prefix. For + /// example, a command-line value of `src/test/ui/abi/variadic-ffi.rs` + /// will match `src/test/ui`. A command-line value of `ui` would also + /// match `src/test/ui`. + Suite(TaskPath), +} + +impl PathSet { + fn empty() -> PathSet { + PathSet::Set(BTreeSet::new()) + } + + fn one<P: Into<PathBuf>>(path: P, kind: Kind) -> PathSet { + let mut set = BTreeSet::new(); + set.insert(TaskPath { path: path.into(), kind: Some(kind) }); + PathSet::Set(set) + } + + fn has(&self, needle: &Path, module: Option<Kind>) -> bool { + match self { + PathSet::Set(set) => set.iter().any(|p| Self::check(p, needle, module)), + PathSet::Suite(suite) => Self::check(suite, needle, module), + } + } + + // internal use only + fn check(p: &TaskPath, needle: &Path, module: Option<Kind>) -> bool { + if let (Some(p_kind), Some(kind)) = (&p.kind, module) { + p.path.ends_with(needle) && *p_kind == kind + } else { + p.path.ends_with(needle) + } + } + + /// Return all `TaskPath`s in `Self` that contain any of the `needles`, removing the + /// matched needles. + /// + /// This is used for `StepDescription::krate`, which passes all matching crates at once to + /// `Step::make_run`, rather than calling it many times with a single crate. + /// See `tests.rs` for examples. + fn intersection_removing_matches( + &self, + needles: &mut Vec<&Path>, + module: Option<Kind>, + ) -> PathSet { + let mut check = |p| { + for (i, n) in needles.iter().enumerate() { + let matched = Self::check(p, n, module); + if matched { + needles.remove(i); + return true; + } + } + false + }; + match self { + PathSet::Set(set) => PathSet::Set(set.iter().filter(|&p| check(p)).cloned().collect()), + PathSet::Suite(suite) => { + if check(suite) { + self.clone() + } else { + PathSet::empty() + } + } + } + } + + /// A convenience wrapper for Steps which know they have no aliases and all their sets contain only a single path. + /// + /// This can be used with [`ShouldRun::krate`], [`ShouldRun::path`], or [`ShouldRun::alias`]. + #[track_caller] + pub fn assert_single_path(&self) -> &TaskPath { + match self { + PathSet::Set(set) => { + assert_eq!(set.len(), 1, "called assert_single_path on multiple paths"); + set.iter().next().unwrap() + } + PathSet::Suite(_) => unreachable!("called assert_single_path on a Suite path"), + } + } +} + +impl StepDescription { + fn from<S: Step>(kind: Kind) -> StepDescription { + StepDescription { + default: S::DEFAULT, + only_hosts: S::ONLY_HOSTS, + should_run: S::should_run, + make_run: S::make_run, + name: std::any::type_name::<S>(), + kind, + } + } + + fn maybe_run(&self, builder: &Builder<'_>, pathsets: Vec<PathSet>) { + if pathsets.iter().any(|set| self.is_excluded(builder, set)) { + return; + } + + // Determine the targets participating in this rule. + let targets = if self.only_hosts { &builder.hosts } else { &builder.targets }; + + for target in targets { + let run = RunConfig { builder, paths: pathsets.clone(), target: *target }; + (self.make_run)(run); + } + } + + fn is_excluded(&self, builder: &Builder<'_>, pathset: &PathSet) -> bool { + if builder.config.exclude.iter().any(|e| pathset.has(&e.path, e.kind)) { + println!("Skipping {:?} because it is excluded", pathset); + return true; + } + + if !builder.config.exclude.is_empty() { + builder.verbose(&format!( + "{:?} not skipped for {:?} -- not in {:?}", + pathset, self.name, builder.config.exclude + )); + } + false + } + + fn run(v: &[StepDescription], builder: &Builder<'_>, paths: &[PathBuf]) { + let should_runs = v + .iter() + .map(|desc| (desc.should_run)(ShouldRun::new(builder, desc.kind))) + .collect::<Vec<_>>(); + + // sanity checks on rules + for (desc, should_run) in v.iter().zip(&should_runs) { + assert!( + !should_run.paths.is_empty(), + "{:?} should have at least one pathset", + desc.name + ); + } + + if paths.is_empty() || builder.config.include_default_paths { + for (desc, should_run) in v.iter().zip(&should_runs) { + if desc.default && should_run.is_really_default() { + desc.maybe_run(builder, should_run.paths.iter().cloned().collect()); + } + } + } + + // strip CurDir prefix if present + let mut paths: Vec<_> = + paths.into_iter().map(|p| p.strip_prefix(".").unwrap_or(p)).collect(); + + // Handle all test suite paths. + // (This is separate from the loop below to avoid having to handle multiple paths in `is_suite_path` somehow.) + paths.retain(|path| { + for (desc, should_run) in v.iter().zip(&should_runs) { + if let Some(suite) = should_run.is_suite_path(&path) { + desc.maybe_run(builder, vec![suite.clone()]); + return false; + } + } + true + }); + + if paths.is_empty() { + return; + } + + // Handle all PathSets. + for (desc, should_run) in v.iter().zip(&should_runs) { + let pathsets = should_run.pathset_for_paths_removing_matches(&mut paths, desc.kind); + if !pathsets.is_empty() { + desc.maybe_run(builder, pathsets); + } + } + + if !paths.is_empty() { + eprintln!("error: no `{}` rules matched {:?}", builder.kind.as_str(), paths,); + eprintln!( + "help: run `x.py {} --help --verbose` to show a list of available paths", + builder.kind.as_str() + ); + eprintln!( + "note: if you are adding a new Step to bootstrap itself, make sure you register it with `describe!`" + ); + crate::detail_exit(1); + } + } +} + +enum ReallyDefault<'a> { + Bool(bool), + Lazy(Lazy<bool, Box<dyn Fn() -> bool + 'a>>), +} + +pub struct ShouldRun<'a> { + pub builder: &'a Builder<'a>, + kind: Kind, + + // use a BTreeSet to maintain sort order + paths: BTreeSet<PathSet>, + + // If this is a default rule, this is an additional constraint placed on + // its run. Generally something like compiler docs being enabled. + is_really_default: ReallyDefault<'a>, +} + +impl<'a> ShouldRun<'a> { + fn new(builder: &'a Builder<'_>, kind: Kind) -> ShouldRun<'a> { + ShouldRun { + builder, + kind, + paths: BTreeSet::new(), + is_really_default: ReallyDefault::Bool(true), // by default no additional conditions + } + } + + pub fn default_condition(mut self, cond: bool) -> Self { + self.is_really_default = ReallyDefault::Bool(cond); + self + } + + pub fn lazy_default_condition(mut self, lazy_cond: Box<dyn Fn() -> bool + 'a>) -> Self { + self.is_really_default = ReallyDefault::Lazy(Lazy::new(lazy_cond)); + self + } + + pub fn is_really_default(&self) -> bool { + match &self.is_really_default { + ReallyDefault::Bool(val) => *val, + ReallyDefault::Lazy(lazy) => *lazy.deref(), + } + } + + /// Indicates it should run if the command-line selects the given crate or + /// any of its (local) dependencies. + /// + /// Compared to `krate`, this treats the dependencies as aliases for the + /// same job. Generally it is preferred to use `krate`, and treat each + /// individual path separately. For example `./x.py test src/liballoc` + /// (which uses `krate`) will test just `liballoc`. However, `./x.py check + /// src/liballoc` (which uses `all_krates`) will check all of `libtest`. + /// `all_krates` should probably be removed at some point. + pub fn all_krates(mut self, name: &str) -> Self { + let mut set = BTreeSet::new(); + for krate in self.builder.in_tree_crates(name, None) { + let path = krate.local_path(self.builder); + set.insert(TaskPath { path, kind: Some(self.kind) }); + } + self.paths.insert(PathSet::Set(set)); + self + } + + /// Indicates it should run if the command-line selects the given crate or + /// any of its (local) dependencies. + /// + /// `make_run` will be called a single time with all matching command-line paths. + pub fn crate_or_deps(self, name: &str) -> Self { + let crates = self.builder.in_tree_crates(name, None); + self.crates(crates) + } + + /// Indicates it should run if the command-line selects any of the given crates. + /// + /// `make_run` will be called a single time with all matching command-line paths. + pub(crate) fn crates(mut self, crates: Vec<&Crate>) -> Self { + for krate in crates { + let path = krate.local_path(self.builder); + self.paths.insert(PathSet::one(path, self.kind)); + } + self + } + + // single alias, which does not correspond to any on-disk path + pub fn alias(mut self, alias: &str) -> Self { + assert!( + !self.builder.src.join(alias).exists(), + "use `builder.path()` for real paths: {}", + alias + ); + self.paths.insert(PathSet::Set( + std::iter::once(TaskPath { path: alias.into(), kind: Some(self.kind) }).collect(), + )); + self + } + + // single, non-aliased path + pub fn path(self, path: &str) -> Self { + self.paths(&[path]) + } + + // multiple aliases for the same job + pub fn paths(mut self, paths: &[&str]) -> Self { + self.paths.insert(PathSet::Set( + paths + .iter() + .map(|p| { + // FIXME(#96188): make sure this is actually a path. + // This currently breaks for paths within submodules. + //assert!( + // self.builder.src.join(p).exists(), + // "`should_run.paths` should correspond to real on-disk paths - use `alias` if there is no relevant path: {}", + // p + //); + TaskPath { path: p.into(), kind: Some(self.kind) } + }) + .collect(), + )); + self + } + + /// Handles individual files (not directories) within a test suite. + fn is_suite_path(&self, requested_path: &Path) -> Option<&PathSet> { + self.paths.iter().find(|pathset| match pathset { + PathSet::Suite(suite) => requested_path.starts_with(&suite.path), + PathSet::Set(_) => false, + }) + } + + pub fn suite_path(mut self, suite: &str) -> Self { + self.paths.insert(PathSet::Suite(TaskPath { path: suite.into(), kind: Some(self.kind) })); + self + } + + // allows being more explicit about why should_run in Step returns the value passed to it + pub fn never(mut self) -> ShouldRun<'a> { + self.paths.insert(PathSet::empty()); + self + } + + /// Given a set of requested paths, return the subset which match the Step for this `ShouldRun`, + /// removing the matches from `paths`. + /// + /// NOTE: this returns multiple PathSets to allow for the possibility of multiple units of work + /// within the same step. For example, `test::Crate` allows testing multiple crates in the same + /// cargo invocation, which are put into separate sets because they aren't aliases. + /// + /// The reason we return PathSet instead of PathBuf is to allow for aliases that mean the same thing + /// (for now, just `all_krates` and `paths`, but we may want to add an `aliases` function in the future?) + fn pathset_for_paths_removing_matches( + &self, + paths: &mut Vec<&Path>, + kind: Kind, + ) -> Vec<PathSet> { + let mut sets = vec![]; + for pathset in &self.paths { + let subset = pathset.intersection_removing_matches(paths, Some(kind)); + if subset != PathSet::empty() { + sets.push(subset); + } + } + sets + } +} + +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug)] +pub enum Kind { + Build, + Check, + Clippy, + Fix, + Format, + Test, + Bench, + Doc, + Clean, + Dist, + Install, + Run, + Setup, +} + +impl Kind { + pub fn parse(string: &str) -> Option<Kind> { + // these strings, including the one-letter aliases, must match the x.py help text + Some(match string { + "build" | "b" => Kind::Build, + "check" | "c" => Kind::Check, + "clippy" => Kind::Clippy, + "fix" => Kind::Fix, + "fmt" => Kind::Format, + "test" | "t" => Kind::Test, + "bench" => Kind::Bench, + "doc" | "d" => Kind::Doc, + "clean" => Kind::Clean, + "dist" => Kind::Dist, + "install" => Kind::Install, + "run" | "r" => Kind::Run, + "setup" => Kind::Setup, + _ => return None, + }) + } + + pub fn as_str(&self) -> &'static str { + match self { + Kind::Build => "build", + Kind::Check => "check", + Kind::Clippy => "clippy", + Kind::Fix => "fix", + Kind::Format => "fmt", + Kind::Test => "test", + Kind::Bench => "bench", + Kind::Doc => "doc", + Kind::Clean => "clean", + Kind::Dist => "dist", + Kind::Install => "install", + Kind::Run => "run", + Kind::Setup => "setup", + } + } +} + +impl<'a> Builder<'a> { + fn get_step_descriptions(kind: Kind) -> Vec<StepDescription> { + macro_rules! describe { + ($($rule:ty),+ $(,)?) => {{ + vec![$(StepDescription::from::<$rule>(kind)),+] + }}; + } + match kind { + Kind::Build => describe!( + compile::Std, + compile::Rustc, + compile::Assemble, + compile::CodegenBackend, + compile::StartupObjects, + tool::BuildManifest, + tool::Rustbook, + tool::ErrorIndex, + tool::UnstableBookGen, + tool::Tidy, + tool::Linkchecker, + tool::CargoTest, + tool::Compiletest, + tool::RemoteTestServer, + tool::RemoteTestClient, + tool::RustInstaller, + tool::Cargo, + tool::Rls, + tool::RustAnalyzer, + tool::RustAnalyzerProcMacroSrv, + tool::RustDemangler, + tool::Rustdoc, + tool::Clippy, + tool::CargoClippy, + native::Llvm, + native::Sanitizers, + tool::Rustfmt, + tool::Miri, + tool::CargoMiri, + native::Lld, + native::CrtBeginEnd + ), + Kind::Check | Kind::Clippy | Kind::Fix => describe!( + check::Std, + check::Rustc, + check::Rustdoc, + check::CodegenBackend, + check::Clippy, + check::Miri, + check::Rls, + check::RustAnalyzer, + check::Rustfmt, + check::Bootstrap + ), + Kind::Test => describe!( + crate::toolstate::ToolStateCheck, + test::ExpandYamlAnchors, + test::Tidy, + test::Ui, + test::RunPassValgrind, + test::MirOpt, + test::Codegen, + test::CodegenUnits, + test::Assembly, + test::Incremental, + test::Debuginfo, + test::UiFullDeps, + test::Rustdoc, + test::Pretty, + test::Crate, + test::CrateLibrustc, + test::CrateRustdoc, + test::CrateRustdocJsonTypes, + test::Linkcheck, + test::TierCheck, + test::Cargotest, + test::Cargo, + test::Rls, + test::RustAnalyzer, + test::ErrorIndex, + test::Distcheck, + test::RunMakeFullDeps, + test::Nomicon, + test::Reference, + test::RustdocBook, + test::RustByExample, + test::TheBook, + test::UnstableBook, + test::RustcBook, + test::LintDocs, + test::RustcGuide, + test::EmbeddedBook, + test::EditionGuide, + test::Rustfmt, + test::Miri, + test::Clippy, + test::RustDemangler, + test::CompiletestTest, + test::RustdocJSStd, + test::RustdocJSNotStd, + test::RustdocGUI, + test::RustdocTheme, + test::RustdocUi, + test::RustdocJson, + test::HtmlCheck, + // Run bootstrap close to the end as it's unlikely to fail + test::Bootstrap, + // Run run-make last, since these won't pass without make on Windows + test::RunMake, + ), + Kind::Bench => describe!(test::Crate, test::CrateLibrustc), + Kind::Doc => describe!( + doc::UnstableBook, + doc::UnstableBookGen, + doc::TheBook, + doc::Standalone, + doc::Std, + doc::Rustc, + doc::Rustdoc, + doc::Rustfmt, + doc::ErrorIndex, + doc::Nomicon, + doc::Reference, + doc::RustdocBook, + doc::RustByExample, + doc::RustcBook, + doc::CargoBook, + doc::Clippy, + doc::ClippyBook, + doc::Miri, + doc::EmbeddedBook, + doc::EditionGuide, + ), + Kind::Dist => describe!( + dist::Docs, + dist::RustcDocs, + dist::Mingw, + dist::Rustc, + dist::Std, + dist::RustcDev, + dist::Analysis, + dist::Src, + dist::Cargo, + dist::Rls, + dist::RustAnalyzer, + dist::Rustfmt, + dist::RustDemangler, + dist::Clippy, + dist::Miri, + dist::LlvmTools, + dist::RustDev, + dist::Extended, + // It seems that PlainSourceTarball somehow changes how some of the tools + // perceive their dependencies (see #93033) which would invalidate fingerprints + // and force us to rebuild tools after vendoring dependencies. + // To work around this, create the Tarball after building all the tools. + dist::PlainSourceTarball, + dist::BuildManifest, + dist::ReproducibleArtifacts, + ), + Kind::Install => describe!( + install::Docs, + install::Std, + install::Cargo, + install::Rls, + install::RustAnalyzer, + install::Rustfmt, + install::RustDemangler, + install::Clippy, + install::Miri, + install::Analysis, + install::Src, + install::Rustc + ), + Kind::Run => describe!(run::ExpandYamlAnchors, run::BuildManifest, run::BumpStage0), + // These commands either don't use paths, or they're special-cased in Build::build() + Kind::Clean | Kind::Format | Kind::Setup => vec![], + } + } + + pub fn get_help(build: &Build, kind: Kind) -> Option<String> { + let step_descriptions = Builder::get_step_descriptions(kind); + if step_descriptions.is_empty() { + return None; + } + + let builder = Self::new_internal(build, kind, vec![]); + let builder = &builder; + // The "build" kind here is just a placeholder, it will be replaced with something else in + // the following statement. + let mut should_run = ShouldRun::new(builder, Kind::Build); + for desc in step_descriptions { + should_run.kind = desc.kind; + should_run = (desc.should_run)(should_run); + } + let mut help = String::from("Available paths:\n"); + let mut add_path = |path: &Path| { + t!(write!(help, " ./x.py {} {}\n", kind.as_str(), path.display())); + }; + for pathset in should_run.paths { + match pathset { + PathSet::Set(set) => { + for path in set { + add_path(&path.path); + } + } + PathSet::Suite(path) => { + add_path(&path.path.join("...")); + } + } + } + Some(help) + } + + fn new_internal(build: &Build, kind: Kind, paths: Vec<PathBuf>) -> Builder<'_> { + Builder { + build, + top_stage: build.config.stage, + kind, + cache: Cache::new(), + stack: RefCell::new(Vec::new()), + time_spent_on_dependencies: Cell::new(Duration::new(0, 0)), + paths, + } + } + + pub fn new(build: &Build) -> Builder<'_> { + let (kind, paths) = match build.config.cmd { + Subcommand::Build { ref paths } => (Kind::Build, &paths[..]), + Subcommand::Check { ref paths } => (Kind::Check, &paths[..]), + Subcommand::Clippy { ref paths, .. } => (Kind::Clippy, &paths[..]), + Subcommand::Fix { ref paths } => (Kind::Fix, &paths[..]), + Subcommand::Doc { ref paths, .. } => (Kind::Doc, &paths[..]), + Subcommand::Test { ref paths, .. } => (Kind::Test, &paths[..]), + Subcommand::Bench { ref paths, .. } => (Kind::Bench, &paths[..]), + Subcommand::Dist { ref paths } => (Kind::Dist, &paths[..]), + Subcommand::Install { ref paths } => (Kind::Install, &paths[..]), + Subcommand::Run { ref paths } => (Kind::Run, &paths[..]), + Subcommand::Format { .. } => (Kind::Format, &[][..]), + Subcommand::Clean { .. } | Subcommand::Setup { .. } => { + panic!() + } + }; + + Self::new_internal(build, kind, paths.to_owned()) + } + + pub fn execute_cli(&self) { + self.run_step_descriptions(&Builder::get_step_descriptions(self.kind), &self.paths); + } + + pub fn default_doc(&self, paths: &[PathBuf]) { + self.run_step_descriptions(&Builder::get_step_descriptions(Kind::Doc), paths); + } + + /// NOTE: keep this in sync with `rustdoc::clean::utils::doc_rust_lang_org_channel`, or tests will fail on beta/stable. + pub fn doc_rust_lang_org_channel(&self) -> String { + let channel = match &*self.config.channel { + "stable" => &self.version, + "beta" => "beta", + "nightly" | "dev" => "nightly", + // custom build of rustdoc maybe? link to the latest stable docs just in case + _ => "stable", + }; + "https://doc.rust-lang.org/".to_owned() + channel + } + + fn run_step_descriptions(&self, v: &[StepDescription], paths: &[PathBuf]) { + StepDescription::run(v, self, paths); + } + + /// Modifies the interpreter section of 'fname' to fix the dynamic linker, + /// or the RPATH section, to fix the dynamic library search path + /// + /// This is only required on NixOS and uses the PatchELF utility to + /// change the interpreter/RPATH of ELF executables. + /// + /// Please see https://nixos.org/patchelf.html for more information + pub(crate) fn fix_bin_or_dylib(&self, fname: &Path) { + // FIXME: cache NixOS detection? + match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() { + Err(_) => return, + Ok(output) if !output.status.success() => return, + Ok(output) => { + let mut s = output.stdout; + if s.last() == Some(&b'\n') { + s.pop(); + } + if s != b"Linux" { + return; + } + } + } + + // If the user has asked binaries to be patched for Nix, then + // don't check for NixOS or `/lib`, just continue to the patching. + // NOTE: this intentionally comes after the Linux check: + // - patchelf only works with ELF files, so no need to run it on Mac or Windows + // - On other Unix systems, there is no stable syscall interface, so Nix doesn't manage the global libc. + if !self.config.patch_binaries_for_nix { + // Use `/etc/os-release` instead of `/etc/NIXOS`. + // The latter one does not exist on NixOS when using tmpfs as root. + const NIX_IDS: &[&str] = &["ID=nixos", "ID='nixos'", "ID=\"nixos\""]; + let os_release = match File::open("/etc/os-release") { + Err(e) if e.kind() == ErrorKind::NotFound => return, + Err(e) => panic!("failed to access /etc/os-release: {}", e), + Ok(f) => f, + }; + if !BufReader::new(os_release).lines().any(|l| NIX_IDS.contains(&t!(l).trim())) { + return; + } + if Path::new("/lib").exists() { + return; + } + } + + // At this point we're pretty sure the user is running NixOS or using Nix + println!("info: you seem to be using Nix. Attempting to patch {}", fname.display()); + + // Only build `.nix-deps` once. + static NIX_DEPS_DIR: OnceCell<PathBuf> = OnceCell::new(); + let mut nix_build_succeeded = true; + let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| { + // Run `nix-build` to "build" each dependency (which will likely reuse + // the existing `/nix/store` copy, or at most download a pre-built copy). + // + // Importantly, we create a gc-root called `.nix-deps` in the `build/` + // directory, but still reference the actual `/nix/store` path in the rpath + // as it makes it significantly more robust against changes to the location of + // the `.nix-deps` location. + // + // bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`). + // zlib: Needed as a system dependency of `libLLVM-*.so`. + // patchelf: Needed for patching ELF binaries (see doc comment above). + let nix_deps_dir = self.out.join(".nix-deps"); + const NIX_EXPR: &str = " + with (import <nixpkgs> {}); + symlinkJoin { + name = \"rust-stage0-dependencies\"; + paths = [ + zlib + patchelf + stdenv.cc.bintools + ]; + } + "; + nix_build_succeeded = self.try_run(Command::new("nix-build").args(&[ + Path::new("-E"), + Path::new(NIX_EXPR), + Path::new("-o"), + &nix_deps_dir, + ])); + nix_deps_dir + }); + if !nix_build_succeeded { + return; + } + + let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf")); + let rpath_entries = { + // ORIGIN is a relative default, all binary and dynamic libraries we ship + // appear to have this (even when `../lib` is redundant). + // NOTE: there are only two paths here, delimited by a `:` + let mut entries = OsString::from("$ORIGIN/../lib:"); + entries.push(t!(fs::canonicalize(nix_deps_dir))); + entries.push("/lib"); + entries + }; + patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]); + if !fname.extension().map_or(false, |ext| ext == "so") { + // Finally, set the corret .interp for binaries + let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker"); + // FIXME: can we support utf8 here? `args` doesn't accept Vec<u8>, only OsString ... + let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path)))); + patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]); + } + + self.try_run(patchelf.arg(fname)); + } + + pub(crate) fn download_component(&self, url: &str, dest_path: &Path, help_on_error: &str) { + self.verbose(&format!("download {url}")); + // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/. + let tempfile = self.tempdir().join(dest_path.file_name().unwrap()); + // While bootstrap itself only supports http and https downloads, downstream forks might + // need to download components from other protocols. The match allows them adding more + // protocols without worrying about merge conficts if we change the HTTP implementation. + match url.split_once("://").map(|(proto, _)| proto) { + Some("http") | Some("https") => { + self.download_http_with_retries(&tempfile, url, help_on_error) + } + Some(other) => panic!("unsupported protocol {other} in {url}"), + None => panic!("no protocol in {url}"), + } + t!(std::fs::rename(&tempfile, dest_path)); + } + + fn download_http_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) { + println!("downloading {}", url); + // Try curl. If that fails and we are on windows, fallback to PowerShell. + let mut curl = Command::new("curl"); + curl.args(&[ + "-#", + "-y", + "30", + "-Y", + "10", // timeout if speed is < 10 bytes/sec for > 30 seconds + "--connect-timeout", + "30", // timeout if cannot connect within 30 seconds + "--retry", + "3", + "-Sf", + "-o", + ]); + curl.arg(tempfile); + curl.arg(url); + if !self.check_run(&mut curl) { + if self.build.build.contains("windows-msvc") { + println!("Fallback to PowerShell"); + for _ in 0..3 { + if self.try_run(Command::new("PowerShell.exe").args(&[ + "/nologo", + "-Command", + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", + &format!( + "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", + url, tempfile.to_str().expect("invalid UTF-8 not supported with powershell downloads"), + ), + ])) { + return; + } + println!("\nspurious failure, trying again"); + } + } + if !help_on_error.is_empty() { + eprintln!("{}", help_on_error); + } + crate::detail_exit(1); + } + } + + pub(crate) fn unpack(&self, tarball: &Path, dst: &Path, pattern: &str) { + println!("extracting {} to {}", tarball.display(), dst.display()); + if !dst.exists() { + t!(fs::create_dir_all(dst)); + } + + // `tarball` ends with `.tar.xz`; strip that suffix + // example: `rust-dev-nightly-x86_64-unknown-linux-gnu` + let uncompressed_filename = + Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap(); + let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap()); + + // decompress the file + let data = t!(File::open(tarball)); + let decompressor = XzDecoder::new(BufReader::new(data)); + + let mut tar = tar::Archive::new(decompressor); + for member in t!(tar.entries()) { + let mut member = t!(member); + let original_path = t!(member.path()).into_owned(); + // skip the top-level directory + if original_path == directory_prefix { + continue; + } + let mut short_path = t!(original_path.strip_prefix(directory_prefix)); + if !short_path.starts_with(pattern) { + continue; + } + short_path = t!(short_path.strip_prefix(pattern)); + let dst_path = dst.join(short_path); + self.verbose(&format!("extracting {} to {}", original_path.display(), dst.display())); + if !t!(member.unpack_in(dst)) { + panic!("path traversal attack ??"); + } + let src_path = dst.join(original_path); + if src_path.is_dir() && dst_path.exists() { + continue; + } + t!(fs::rename(src_path, dst_path)); + } + t!(fs::remove_dir_all(dst.join(directory_prefix))); + } + + /// Returns whether the SHA256 checksum of `path` matches `expected`. + pub(crate) fn verify(&self, path: &Path, expected: &str) -> bool { + use sha2::Digest; + + self.verbose(&format!("verifying {}", path.display())); + let mut hasher = sha2::Sha256::new(); + // FIXME: this is ok for rustfmt (4.1 MB large at time of writing), but it seems memory-intensive for rustc and larger components. + // Consider using streaming IO instead? + let contents = if self.config.dry_run { vec![] } else { t!(fs::read(path)) }; + hasher.update(&contents); + let found = hex::encode(hasher.finalize().as_slice()); + let verified = found == expected; + if !verified && !self.config.dry_run { + println!( + "invalid checksum: \n\ + found: {found}\n\ + expected: {expected}", + ); + } + return verified; + } + + /// Obtain a compiler at a given stage and for a given host. Explicitly does + /// not take `Compiler` since all `Compiler` instances are meant to be + /// obtained through this function, since it ensures that they are valid + /// (i.e., built and assembled). + pub fn compiler(&self, stage: u32, host: TargetSelection) -> Compiler { + self.ensure(compile::Assemble { target_compiler: Compiler { stage, host } }) + } + + /// Similar to `compiler`, except handles the full-bootstrap option to + /// silently use the stage1 compiler instead of a stage2 compiler if one is + /// requested. + /// + /// Note that this does *not* have the side effect of creating + /// `compiler(stage, host)`, unlike `compiler` above which does have such + /// a side effect. The returned compiler here can only be used to compile + /// new artifacts, it can't be used to rely on the presence of a particular + /// sysroot. + /// + /// See `force_use_stage1` for documentation on what each argument is. + pub fn compiler_for( + &self, + stage: u32, + host: TargetSelection, + target: TargetSelection, + ) -> Compiler { + if self.build.force_use_stage1(Compiler { stage, host }, target) { + self.compiler(1, self.config.build) + } else { + self.compiler(stage, host) + } + } + + pub fn sysroot(&self, compiler: Compiler) -> Interned<PathBuf> { + self.ensure(compile::Sysroot { compiler }) + } + + /// Returns the libdir where the standard library and other artifacts are + /// found for a compiler's sysroot. + pub fn sysroot_libdir(&self, compiler: Compiler, target: TargetSelection) -> Interned<PathBuf> { + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] + struct Libdir { + compiler: Compiler, + target: TargetSelection, + } + impl Step for Libdir { + type Output = Interned<PathBuf>; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + fn run(self, builder: &Builder<'_>) -> Interned<PathBuf> { + let lib = builder.sysroot_libdir_relative(self.compiler); + let sysroot = builder + .sysroot(self.compiler) + .join(lib) + .join("rustlib") + .join(self.target.triple) + .join("lib"); + // Avoid deleting the rustlib/ directory we just copied + // (in `impl Step for Sysroot`). + if !builder.download_rustc() { + let _ = fs::remove_dir_all(&sysroot); + t!(fs::create_dir_all(&sysroot)); + } + INTERNER.intern_path(sysroot) + } + } + self.ensure(Libdir { compiler, target }) + } + + pub fn sysroot_codegen_backends(&self, compiler: Compiler) -> PathBuf { + self.sysroot_libdir(compiler, compiler.host).with_file_name("codegen-backends") + } + + /// Returns the compiler's libdir where it stores the dynamic libraries that + /// it itself links against. + /// + /// For example this returns `<sysroot>/lib` on Unix and `<sysroot>/bin` on + /// Windows. + pub fn rustc_libdir(&self, compiler: Compiler) -> PathBuf { + if compiler.is_snapshot(self) { + self.rustc_snapshot_libdir() + } else { + match self.config.libdir_relative() { + Some(relative_libdir) if compiler.stage >= 1 => { + self.sysroot(compiler).join(relative_libdir) + } + _ => self.sysroot(compiler).join(libdir(compiler.host)), + } + } + } + + /// Returns the compiler's relative libdir where it stores the dynamic libraries that + /// it itself links against. + /// + /// For example this returns `lib` on Unix and `bin` on + /// Windows. + pub fn libdir_relative(&self, compiler: Compiler) -> &Path { + if compiler.is_snapshot(self) { + libdir(self.config.build).as_ref() + } else { + match self.config.libdir_relative() { + Some(relative_libdir) if compiler.stage >= 1 => relative_libdir, + _ => libdir(compiler.host).as_ref(), + } + } + } + + /// Returns the compiler's relative libdir where the standard library and other artifacts are + /// found for a compiler's sysroot. + /// + /// For example this returns `lib` on Unix and Windows. + pub fn sysroot_libdir_relative(&self, compiler: Compiler) -> &Path { + match self.config.libdir_relative() { + Some(relative_libdir) if compiler.stage >= 1 => relative_libdir, + _ if compiler.stage == 0 => &self.build.initial_libdir, + _ => Path::new("lib"), + } + } + + pub fn rustc_lib_paths(&self, compiler: Compiler) -> Vec<PathBuf> { + let mut dylib_dirs = vec![self.rustc_libdir(compiler)]; + + // Ensure that the downloaded LLVM libraries can be found. + if self.config.llvm_from_ci { + let ci_llvm_lib = self.out.join(&*compiler.host.triple).join("ci-llvm").join("lib"); + dylib_dirs.push(ci_llvm_lib); + } + + dylib_dirs + } + + /// Adds the compiler's directory of dynamic libraries to `cmd`'s dynamic + /// library lookup path. + pub fn add_rustc_lib_path(&self, compiler: Compiler, cmd: &mut Command) { + // Windows doesn't need dylib path munging because the dlls for the + // compiler live next to the compiler and the system will find them + // automatically. + if cfg!(windows) { + return; + } + + add_dylib_path(self.rustc_lib_paths(compiler), cmd); + } + + /// Gets a path to the compiler specified. + pub fn rustc(&self, compiler: Compiler) -> PathBuf { + if compiler.is_snapshot(self) { + self.initial_rustc.clone() + } else { + self.sysroot(compiler).join("bin").join(exe("rustc", compiler.host)) + } + } + + /// Gets the paths to all of the compiler's codegen backends. + fn codegen_backends(&self, compiler: Compiler) -> impl Iterator<Item = PathBuf> { + fs::read_dir(self.sysroot_codegen_backends(compiler)) + .into_iter() + .flatten() + .filter_map(Result::ok) + .map(|entry| entry.path()) + } + + pub fn rustdoc(&self, compiler: Compiler) -> PathBuf { + self.ensure(tool::Rustdoc { compiler }) + } + + pub fn rustdoc_cmd(&self, compiler: Compiler) -> Command { + let mut cmd = Command::new(&self.bootstrap_out.join("rustdoc")); + cmd.env("RUSTC_STAGE", compiler.stage.to_string()) + .env("RUSTC_SYSROOT", self.sysroot(compiler)) + // Note that this is *not* the sysroot_libdir because rustdoc must be linked + // equivalently to rustc. + .env("RUSTDOC_LIBDIR", self.rustc_libdir(compiler)) + .env("CFG_RELEASE_CHANNEL", &self.config.channel) + .env("RUSTDOC_REAL", self.rustdoc(compiler)) + .env("RUSTC_BOOTSTRAP", "1"); + + cmd.arg("-Wrustdoc::invalid_codeblock_attributes"); + + if self.config.deny_warnings { + cmd.arg("-Dwarnings"); + } + cmd.arg("-Znormalize-docs"); + + // Remove make-related flags that can cause jobserver problems. + cmd.env_remove("MAKEFLAGS"); + cmd.env_remove("MFLAGS"); + + if let Some(linker) = self.linker(compiler.host) { + cmd.env("RUSTDOC_LINKER", linker); + } + if self.is_fuse_ld_lld(compiler.host) { + cmd.env("RUSTDOC_FUSE_LD_LLD", "1"); + } + cmd + } + + /// Return the path to `llvm-config` for the target, if it exists. + /// + /// Note that this returns `None` if LLVM is disabled, or if we're in a + /// check build or dry-run, where there's no need to build all of LLVM. + fn llvm_config(&self, target: TargetSelection) -> Option<PathBuf> { + if self.config.llvm_enabled() && self.kind != Kind::Check && !self.config.dry_run { + let llvm_config = self.ensure(native::Llvm { target }); + if llvm_config.is_file() { + return Some(llvm_config); + } + } + None + } + + /// Convenience wrapper to allow `builder.llvm_link_shared()` instead of `builder.config.llvm_link_shared(&builder)`. + pub(crate) fn llvm_link_shared(&self) -> bool { + Config::llvm_link_shared(self) + } + + pub(crate) fn download_rustc(&self) -> bool { + Config::download_rustc(self) + } + + pub(crate) fn initial_rustfmt(&self) -> Option<PathBuf> { + Config::initial_rustfmt(self) + } + + /// Prepares an invocation of `cargo` to be run. + /// + /// This will create a `Command` that represents a pending execution of + /// Cargo. This cargo will be configured to use `compiler` as the actual + /// rustc compiler, its output will be scoped by `mode`'s output directory, + /// it will pass the `--target` flag for the specified `target`, and will be + /// executing the Cargo command `cmd`. + pub fn cargo( + &self, + compiler: Compiler, + mode: Mode, + source_type: SourceType, + target: TargetSelection, + cmd: &str, + ) -> Cargo { + let mut cargo = Command::new(&self.initial_cargo); + let out_dir = self.stage_out(compiler, mode); + + // Codegen backends are not yet tracked by -Zbinary-dep-depinfo, + // so we need to explicitly clear out if they've been updated. + for backend in self.codegen_backends(compiler) { + self.clear_if_dirty(&out_dir, &backend); + } + + if cmd == "doc" || cmd == "rustdoc" { + let my_out = match mode { + // This is the intended out directory for compiler documentation. + Mode::Rustc | Mode::ToolRustc => self.compiler_doc_out(target), + Mode::Std => out_dir.join(target.triple).join("doc"), + _ => panic!("doc mode {:?} not expected", mode), + }; + let rustdoc = self.rustdoc(compiler); + self.clear_if_dirty(&my_out, &rustdoc); + } + + cargo.env("CARGO_TARGET_DIR", &out_dir).arg(cmd); + + let profile_var = |name: &str| { + let profile = if self.config.rust_optimize { "RELEASE" } else { "DEV" }; + format!("CARGO_PROFILE_{}_{}", profile, name) + }; + + // See comment in rustc_llvm/build.rs for why this is necessary, largely llvm-config + // needs to not accidentally link to libLLVM in stage0/lib. + cargo.env("REAL_LIBRARY_PATH_VAR", &util::dylib_path_var()); + if let Some(e) = env::var_os(util::dylib_path_var()) { + cargo.env("REAL_LIBRARY_PATH", e); + } + + // Found with `rg "init_env_logger\("`. If anyone uses `init_env_logger` + // from out of tree it shouldn't matter, since x.py is only used for + // building in-tree. + let color_logs = ["RUSTDOC_LOG_COLOR", "RUSTC_LOG_COLOR", "RUST_LOG_COLOR"]; + match self.build.config.color { + Color::Always => { + cargo.arg("--color=always"); + for log in &color_logs { + cargo.env(log, "always"); + } + } + Color::Never => { + cargo.arg("--color=never"); + for log in &color_logs { + cargo.env(log, "never"); + } + } + Color::Auto => {} // nothing to do + } + + if cmd != "install" { + cargo.arg("--target").arg(target.rustc_target_arg()); + } else { + assert_eq!(target, compiler.host); + } + + // Set a flag for `check`/`clippy`/`fix`, so that certain build + // scripts can do less work (i.e. not building/requiring LLVM). + if cmd == "check" || cmd == "clippy" || cmd == "fix" { + // If we've not yet built LLVM, or it's stale, then bust + // the rustc_llvm cache. That will always work, even though it + // may mean that on the next non-check build we'll need to rebuild + // rustc_llvm. But if LLVM is stale, that'll be a tiny amount + // of work comparatively, and we'd likely need to rebuild it anyway, + // so that's okay. + if crate::native::prebuilt_llvm_config(self, target).is_err() { + cargo.env("RUST_CHECK", "1"); + } + } + + let stage = if compiler.stage == 0 && self.local_rebuild { + // Assume the local-rebuild rustc already has stage1 features. + 1 + } else { + compiler.stage + }; + + let mut rustflags = Rustflags::new(target); + if stage != 0 { + if let Ok(s) = env::var("CARGOFLAGS_NOT_BOOTSTRAP") { + cargo.args(s.split_whitespace()); + } + rustflags.env("RUSTFLAGS_NOT_BOOTSTRAP"); + } else { + if let Ok(s) = env::var("CARGOFLAGS_BOOTSTRAP") { + cargo.args(s.split_whitespace()); + } + rustflags.env("RUSTFLAGS_BOOTSTRAP"); + if cmd == "clippy" { + // clippy overwrites sysroot if we pass it to cargo. + // Pass it directly to clippy instead. + // NOTE: this can't be fixed in clippy because we explicitly don't set `RUSTC`, + // so it has no way of knowing the sysroot. + rustflags.arg("--sysroot"); + rustflags.arg( + self.sysroot(compiler) + .as_os_str() + .to_str() + .expect("sysroot must be valid UTF-8"), + ); + // Only run clippy on a very limited subset of crates (in particular, not build scripts). + cargo.arg("-Zunstable-options"); + // Explicitly does *not* set `--cfg=bootstrap`, since we're using a nightly clippy. + let host_version = Command::new("rustc").arg("--version").output().map_err(|_| ()); + let output = host_version.and_then(|output| { + if output.status.success() { + Ok(output) + } else { + Err(()) + } + }).unwrap_or_else(|_| { + eprintln!( + "error: `x.py clippy` requires a host `rustc` toolchain with the `clippy` component" + ); + eprintln!("help: try `rustup component add clippy`"); + crate::detail_exit(1); + }); + if !t!(std::str::from_utf8(&output.stdout)).contains("nightly") { + rustflags.arg("--cfg=bootstrap"); + } + } else { + rustflags.arg("--cfg=bootstrap"); + } + } + + let use_new_symbol_mangling = match self.config.rust_new_symbol_mangling { + Some(setting) => { + // If an explicit setting is given, use that + setting + } + None => { + if mode == Mode::Std { + // The standard library defaults to the legacy scheme + false + } else { + // The compiler and tools default to the new scheme + true + } + } + }; + + if use_new_symbol_mangling { + rustflags.arg("-Csymbol-mangling-version=v0"); + } else { + rustflags.arg("-Csymbol-mangling-version=legacy"); + rustflags.arg("-Zunstable-options"); + } + + // Enable cfg checking of cargo features for everything but std and also enable cfg + // checking of names and values. + // + // Note: `std`, `alloc` and `core` imports some dependencies by #[path] (like + // backtrace, core_simd, std_float, ...), those dependencies have their own + // features but cargo isn't involved in the #[path] process and so cannot pass the + // complete list of features, so for that reason we don't enable checking of + // features for std crates. + cargo.arg(if mode != Mode::Std { + "-Zcheck-cfg=names,values,output,features" + } else { + "-Zcheck-cfg=names,values,output" + }); + + // Add extra cfg not defined in/by rustc + // + // Note: Altrough it would seems that "-Zunstable-options" to `rustflags` is useless as + // cargo would implicitly add it, it was discover that sometimes bootstrap only use + // `rustflags` without `cargo` making it required. + rustflags.arg("-Zunstable-options"); + for (restricted_mode, name, values) in EXTRA_CHECK_CFGS { + if *restricted_mode == None || *restricted_mode == Some(mode) { + // Creating a string of the values by concatenating each value: + // ',"tvos","watchos"' or '' (nothing) when there are no values + let values = match values { + Some(values) => values + .iter() + .map(|val| [",", "\"", val, "\""]) + .flatten() + .collect::<String>(), + None => String::new(), + }; + rustflags.arg(&format!("--check-cfg=values({name}{values})")); + } + } + + // FIXME: It might be better to use the same value for both `RUSTFLAGS` and `RUSTDOCFLAGS`, + // but this breaks CI. At the very least, stage0 `rustdoc` needs `--cfg bootstrap`. See + // #71458. + let mut rustdocflags = rustflags.clone(); + rustdocflags.propagate_cargo_env("RUSTDOCFLAGS"); + if stage == 0 { + rustdocflags.env("RUSTDOCFLAGS_BOOTSTRAP"); + } else { + rustdocflags.env("RUSTDOCFLAGS_NOT_BOOTSTRAP"); + } + + if let Ok(s) = env::var("CARGOFLAGS") { + cargo.args(s.split_whitespace()); + } + + match mode { + Mode::Std | Mode::ToolBootstrap | Mode::ToolStd => {} + Mode::Rustc | Mode::Codegen | Mode::ToolRustc => { + // Build proc macros both for the host and the target + if target != compiler.host && cmd != "check" { + cargo.arg("-Zdual-proc-macros"); + rustflags.arg("-Zdual-proc-macros"); + } + } + } + + // This tells Cargo (and in turn, rustc) to output more complete + // dependency information. Most importantly for rustbuild, this + // includes sysroot artifacts, like libstd, which means that we don't + // need to track those in rustbuild (an error prone process!). This + // feature is currently unstable as there may be some bugs and such, but + // it represents a big improvement in rustbuild's reliability on + // rebuilds, so we're using it here. + // + // For some additional context, see #63470 (the PR originally adding + // this), as well as #63012 which is the tracking issue for this + // feature on the rustc side. + cargo.arg("-Zbinary-dep-depinfo"); + match mode { + Mode::ToolBootstrap => { + // Restrict the allowed features to those passed by rustbuild, so we don't depend on nightly accidentally. + // HACK: because anyhow does feature detection in build.rs, we need to allow the backtrace feature too. + rustflags.arg("-Zallow-features=binary-dep-depinfo,backtrace"); + } + Mode::ToolStd => { + // Right now this is just compiletest and a few other tools that build on stable. + // Allow them to use `feature(test)`, but nothing else. + rustflags.arg("-Zallow-features=binary-dep-depinfo,test,backtrace,proc_macro_internals,proc_macro_diagnostic,proc_macro_span"); + } + Mode::Std | Mode::Rustc | Mode::Codegen | Mode::ToolRustc => {} + } + + cargo.arg("-j").arg(self.jobs().to_string()); + // Remove make-related flags to ensure Cargo can correctly set things up + cargo.env_remove("MAKEFLAGS"); + cargo.env_remove("MFLAGS"); + + // FIXME: Temporary fix for https://github.com/rust-lang/cargo/issues/3005 + // Force cargo to output binaries with disambiguating hashes in the name + let mut metadata = if compiler.stage == 0 { + // Treat stage0 like a special channel, whether it's a normal prior- + // release rustc or a local rebuild with the same version, so we + // never mix these libraries by accident. + "bootstrap".to_string() + } else { + self.config.channel.to_string() + }; + // We want to make sure that none of the dependencies between + // std/test/rustc unify with one another. This is done for weird linkage + // reasons but the gist of the problem is that if librustc, libtest, and + // libstd all depend on libc from crates.io (which they actually do) we + // want to make sure they all get distinct versions. Things get really + // weird if we try to unify all these dependencies right now, namely + // around how many times the library is linked in dynamic libraries and + // such. If rustc were a static executable or if we didn't ship dylibs + // this wouldn't be a problem, but we do, so it is. This is in general + // just here to make sure things build right. If you can remove this and + // things still build right, please do! + match mode { + Mode::Std => metadata.push_str("std"), + // When we're building rustc tools, they're built with a search path + // that contains things built during the rustc build. For example, + // bitflags is built during the rustc build, and is a dependency of + // rustdoc as well. We're building rustdoc in a different target + // directory, though, which means that Cargo will rebuild the + // dependency. When we go on to build rustdoc, we'll look for + // bitflags, and find two different copies: one built during the + // rustc step and one that we just built. This isn't always a + // problem, somehow -- not really clear why -- but we know that this + // fixes things. + Mode::ToolRustc => metadata.push_str("tool-rustc"), + // Same for codegen backends. + Mode::Codegen => metadata.push_str("codegen"), + _ => {} + } + cargo.env("__CARGO_DEFAULT_LIB_METADATA", &metadata); + + if cmd == "clippy" { + rustflags.arg("-Zforce-unstable-if-unmarked"); + } + + rustflags.arg("-Zmacro-backtrace"); + + let want_rustdoc = self.doc_tests != DocTests::No; + + // We synthetically interpret a stage0 compiler used to build tools as a + // "raw" compiler in that it's the exact snapshot we download. Normally + // the stage0 build means it uses libraries build by the stage0 + // compiler, but for tools we just use the precompiled libraries that + // we've downloaded + let use_snapshot = mode == Mode::ToolBootstrap; + assert!(!use_snapshot || stage == 0 || self.local_rebuild); + + let maybe_sysroot = self.sysroot(compiler); + let sysroot = if use_snapshot { self.rustc_snapshot_sysroot() } else { &maybe_sysroot }; + let libdir = self.rustc_libdir(compiler); + + // Clear the output directory if the real rustc we're using has changed; + // Cargo cannot detect this as it thinks rustc is bootstrap/debug/rustc. + // + // Avoid doing this during dry run as that usually means the relevant + // compiler is not yet linked/copied properly. + // + // Only clear out the directory if we're compiling std; otherwise, we + // should let Cargo take care of things for us (via depdep info) + if !self.config.dry_run && mode == Mode::Std && cmd == "build" { + self.clear_if_dirty(&out_dir, &self.rustc(compiler)); + } + + // Customize the compiler we're running. Specify the compiler to cargo + // as our shim and then pass it some various options used to configure + // how the actual compiler itself is called. + // + // These variables are primarily all read by + // src/bootstrap/bin/{rustc.rs,rustdoc.rs} + cargo + .env("RUSTBUILD_NATIVE_DIR", self.native_dir(target)) + .env("RUSTC_REAL", self.rustc(compiler)) + .env("RUSTC_STAGE", stage.to_string()) + .env("RUSTC_SYSROOT", &sysroot) + .env("RUSTC_LIBDIR", &libdir) + .env("RUSTDOC", self.bootstrap_out.join("rustdoc")) + .env( + "RUSTDOC_REAL", + if cmd == "doc" || cmd == "rustdoc" || (cmd == "test" && want_rustdoc) { + self.rustdoc(compiler) + } else { + PathBuf::from("/path/to/nowhere/rustdoc/not/required") + }, + ) + .env("RUSTC_ERROR_METADATA_DST", self.extended_error_dir()) + .env("RUSTC_BREAK_ON_ICE", "1"); + // Clippy support is a hack and uses the default `cargo-clippy` in path. + // Don't override RUSTC so that the `cargo-clippy` in path will be run. + if cmd != "clippy" { + cargo.env("RUSTC", self.bootstrap_out.join("rustc")); + } + + // Dealing with rpath here is a little special, so let's go into some + // detail. First off, `-rpath` is a linker option on Unix platforms + // which adds to the runtime dynamic loader path when looking for + // dynamic libraries. We use this by default on Unix platforms to ensure + // that our nightlies behave the same on Windows, that is they work out + // of the box. This can be disabled, of course, but basically that's why + // we're gated on RUSTC_RPATH here. + // + // Ok, so the astute might be wondering "why isn't `-C rpath` used + // here?" and that is indeed a good question to ask. This codegen + // option is the compiler's current interface to generating an rpath. + // Unfortunately it doesn't quite suffice for us. The flag currently + // takes no value as an argument, so the compiler calculates what it + // should pass to the linker as `-rpath`. This unfortunately is based on + // the **compile time** directory structure which when building with + // Cargo will be very different than the runtime directory structure. + // + // All that's a really long winded way of saying that if we use + // `-Crpath` then the executables generated have the wrong rpath of + // something like `$ORIGIN/deps` when in fact the way we distribute + // rustc requires the rpath to be `$ORIGIN/../lib`. + // + // So, all in all, to set up the correct rpath we pass the linker + // argument manually via `-C link-args=-Wl,-rpath,...`. Plus isn't it + // fun to pass a flag to a tool to pass a flag to pass a flag to a tool + // to change a flag in a binary? + if self.config.rust_rpath && util::use_host_linker(target) { + let rpath = if target.contains("apple") { + // Note that we need to take one extra step on macOS to also pass + // `-Wl,-instal_name,@rpath/...` to get things to work right. To + // do that we pass a weird flag to the compiler to get it to do + // so. Note that this is definitely a hack, and we should likely + // flesh out rpath support more fully in the future. + rustflags.arg("-Zosx-rpath-install-name"); + Some("-Wl,-rpath,@loader_path/../lib") + } else if !target.contains("windows") { + rustflags.arg("-Clink-args=-Wl,-z,origin"); + Some("-Wl,-rpath,$ORIGIN/../lib") + } else { + None + }; + if let Some(rpath) = rpath { + rustflags.arg(&format!("-Clink-args={}", rpath)); + } + } + + if let Some(host_linker) = self.linker(compiler.host) { + cargo.env("RUSTC_HOST_LINKER", host_linker); + } + if self.is_fuse_ld_lld(compiler.host) { + cargo.env("RUSTC_HOST_FUSE_LD_LLD", "1"); + cargo.env("RUSTDOC_FUSE_LD_LLD", "1"); + } + + if let Some(target_linker) = self.linker(target) { + let target = crate::envify(&target.triple); + cargo.env(&format!("CARGO_TARGET_{}_LINKER", target), target_linker); + } + if self.is_fuse_ld_lld(target) { + rustflags.arg("-Clink-args=-fuse-ld=lld"); + } + self.lld_flags(target).for_each(|flag| { + rustdocflags.arg(&flag); + }); + + if !(["build", "check", "clippy", "fix", "rustc"].contains(&cmd)) && want_rustdoc { + cargo.env("RUSTDOC_LIBDIR", self.rustc_libdir(compiler)); + } + + let debuginfo_level = match mode { + Mode::Rustc | Mode::Codegen => self.config.rust_debuginfo_level_rustc, + Mode::Std => self.config.rust_debuginfo_level_std, + Mode::ToolBootstrap | Mode::ToolStd | Mode::ToolRustc => { + self.config.rust_debuginfo_level_tools + } + }; + cargo.env(profile_var("DEBUG"), debuginfo_level.to_string()); + cargo.env( + profile_var("DEBUG_ASSERTIONS"), + if mode == Mode::Std { + self.config.rust_debug_assertions_std.to_string() + } else { + self.config.rust_debug_assertions.to_string() + }, + ); + cargo.env( + profile_var("OVERFLOW_CHECKS"), + if mode == Mode::Std { + self.config.rust_overflow_checks_std.to_string() + } else { + self.config.rust_overflow_checks.to_string() + }, + ); + + if !target.contains("windows") { + let needs_unstable_opts = target.contains("linux") + || target.contains("solaris") + || target.contains("windows") + || target.contains("bsd") + || target.contains("dragonfly") + || target.contains("illumos"); + + if needs_unstable_opts { + rustflags.arg("-Zunstable-options"); + } + match self.config.rust_split_debuginfo { + SplitDebuginfo::Packed => rustflags.arg("-Csplit-debuginfo=packed"), + SplitDebuginfo::Unpacked => rustflags.arg("-Csplit-debuginfo=unpacked"), + SplitDebuginfo::Off => rustflags.arg("-Csplit-debuginfo=off"), + }; + } + + if self.config.cmd.bless() { + // Bless `expect!` tests. + cargo.env("UPDATE_EXPECT", "1"); + } + + if !mode.is_tool() { + cargo.env("RUSTC_FORCE_UNSTABLE", "1"); + } + + if let Some(x) = self.crt_static(target) { + if x { + rustflags.arg("-Ctarget-feature=+crt-static"); + } else { + rustflags.arg("-Ctarget-feature=-crt-static"); + } + } + + if let Some(x) = self.crt_static(compiler.host) { + cargo.env("RUSTC_HOST_CRT_STATIC", x.to_string()); + } + + if let Some(map_to) = self.build.debuginfo_map_to(GitRepo::Rustc) { + let map = format!("{}={}", self.build.src.display(), map_to); + cargo.env("RUSTC_DEBUGINFO_MAP", map); + + // `rustc` needs to know the virtual `/rustc/$hash` we're mapping to, + // in order to opportunistically reverse it later. + cargo.env("CFG_VIRTUAL_RUST_SOURCE_BASE_DIR", map_to); + } + + // Enable usage of unstable features + cargo.env("RUSTC_BOOTSTRAP", "1"); + self.add_rust_test_threads(&mut cargo); + + // Almost all of the crates that we compile as part of the bootstrap may + // have a build script, including the standard library. To compile a + // build script, however, it itself needs a standard library! This + // introduces a bit of a pickle when we're compiling the standard + // library itself. + // + // To work around this we actually end up using the snapshot compiler + // (stage0) for compiling build scripts of the standard library itself. + // The stage0 compiler is guaranteed to have a libstd available for use. + // + // For other crates, however, we know that we've already got a standard + // library up and running, so we can use the normal compiler to compile + // build scripts in that situation. + if mode == Mode::Std { + cargo + .env("RUSTC_SNAPSHOT", &self.initial_rustc) + .env("RUSTC_SNAPSHOT_LIBDIR", self.rustc_snapshot_libdir()); + } else { + cargo + .env("RUSTC_SNAPSHOT", self.rustc(compiler)) + .env("RUSTC_SNAPSHOT_LIBDIR", self.rustc_libdir(compiler)); + } + + // Tools that use compiler libraries may inherit the `-lLLVM` link + // requirement, but the `-L` library path is not propagated across + // separate Cargo projects. We can add LLVM's library path to the + // platform-specific environment variable as a workaround. + if mode == Mode::ToolRustc || mode == Mode::Codegen { + if let Some(llvm_config) = self.llvm_config(target) { + let llvm_libdir = output(Command::new(&llvm_config).arg("--libdir")); + add_link_lib_path(vec![llvm_libdir.trim().into()], &mut cargo); + } + } + + // Compile everything except libraries and proc macros with the more + // efficient initial-exec TLS model. This doesn't work with `dlopen`, + // so we can't use it by default in general, but we can use it for tools + // and our own internal libraries. + if !mode.must_support_dlopen() && !target.triple.starts_with("powerpc-") { + rustflags.arg("-Ztls-model=initial-exec"); + } + + if self.config.incremental { + cargo.env("CARGO_INCREMENTAL", "1"); + } else { + // Don't rely on any default setting for incr. comp. in Cargo + cargo.env("CARGO_INCREMENTAL", "0"); + } + + if let Some(ref on_fail) = self.config.on_fail { + cargo.env("RUSTC_ON_FAIL", on_fail); + } + + if self.config.print_step_timings { + cargo.env("RUSTC_PRINT_STEP_TIMINGS", "1"); + } + + if self.config.print_step_rusage { + cargo.env("RUSTC_PRINT_STEP_RUSAGE", "1"); + } + + if self.config.backtrace_on_ice { + cargo.env("RUSTC_BACKTRACE_ON_ICE", "1"); + } + + cargo.env("RUSTC_VERBOSE", self.verbosity.to_string()); + + if source_type == SourceType::InTree { + let mut lint_flags = Vec::new(); + // When extending this list, add the new lints to the RUSTFLAGS of the + // build_bootstrap function of src/bootstrap/bootstrap.py as well as + // some code doesn't go through this `rustc` wrapper. + lint_flags.push("-Wrust_2018_idioms"); + lint_flags.push("-Wunused_lifetimes"); + lint_flags.push("-Wsemicolon_in_expressions_from_macros"); + + if self.config.deny_warnings { + lint_flags.push("-Dwarnings"); + rustdocflags.arg("-Dwarnings"); + } + + // This does not use RUSTFLAGS due to caching issues with Cargo. + // Clippy is treated as an "in tree" tool, but shares the same + // cache as other "submodule" tools. With these options set in + // RUSTFLAGS, that causes *every* shared dependency to be rebuilt. + // By injecting this into the rustc wrapper, this circumvents + // Cargo's fingerprint detection. This is fine because lint flags + // are always ignored in dependencies. Eventually this should be + // fixed via better support from Cargo. + cargo.env("RUSTC_LINT_FLAGS", lint_flags.join(" ")); + + rustdocflags.arg("-Wrustdoc::invalid_codeblock_attributes"); + } + + if mode == Mode::Rustc { + rustflags.arg("-Zunstable-options"); + rustflags.arg("-Wrustc::internal"); + } + + // Throughout the build Cargo can execute a number of build scripts + // compiling C/C++ code and we need to pass compilers, archivers, flags, etc + // obtained previously to those build scripts. + // Build scripts use either the `cc` crate or `configure/make` so we pass + // the options through environment variables that are fetched and understood by both. + // + // FIXME: the guard against msvc shouldn't need to be here + if target.contains("msvc") { + if let Some(ref cl) = self.config.llvm_clang_cl { + cargo.env("CC", cl).env("CXX", cl); + } + } else { + let ccache = self.config.ccache.as_ref(); + let ccacheify = |s: &Path| { + let ccache = match ccache { + Some(ref s) => s, + None => return s.display().to_string(), + }; + // FIXME: the cc-rs crate only recognizes the literal strings + // `ccache` and `sccache` when doing caching compilations, so we + // mirror that here. It should probably be fixed upstream to + // accept a new env var or otherwise work with custom ccache + // vars. + match &ccache[..] { + "ccache" | "sccache" => format!("{} {}", ccache, s.display()), + _ => s.display().to_string(), + } + }; + let cc = ccacheify(&self.cc(target)); + cargo.env(format!("CC_{}", target.triple), &cc); + + let cflags = self.cflags(target, GitRepo::Rustc, CLang::C).join(" "); + cargo.env(format!("CFLAGS_{}", target.triple), &cflags); + + if let Some(ar) = self.ar(target) { + let ranlib = format!("{} s", ar.display()); + cargo + .env(format!("AR_{}", target.triple), ar) + .env(format!("RANLIB_{}", target.triple), ranlib); + } + + if let Ok(cxx) = self.cxx(target) { + let cxx = ccacheify(&cxx); + let cxxflags = self.cflags(target, GitRepo::Rustc, CLang::Cxx).join(" "); + cargo + .env(format!("CXX_{}", target.triple), &cxx) + .env(format!("CXXFLAGS_{}", target.triple), cxxflags); + } + } + + if mode == Mode::Std && self.config.extended && compiler.is_final_stage(self) { + rustflags.arg("-Zsave-analysis"); + cargo.env( + "RUST_SAVE_ANALYSIS_CONFIG", + "{\"output_file\": null,\"full_docs\": false,\ + \"pub_only\": true,\"reachable_only\": false,\ + \"distro_crate\": true,\"signatures\": false,\"borrow_data\": false}", + ); + } + + // If Control Flow Guard is enabled, pass the `control-flow-guard` flag to rustc + // when compiling the standard library, since this might be linked into the final outputs + // produced by rustc. Since this mitigation is only available on Windows, only enable it + // for the standard library in case the compiler is run on a non-Windows platform. + // This is not needed for stage 0 artifacts because these will only be used for building + // the stage 1 compiler. + if cfg!(windows) + && mode == Mode::Std + && self.config.control_flow_guard + && compiler.stage >= 1 + { + rustflags.arg("-Ccontrol-flow-guard"); + } + + // For `cargo doc` invocations, make rustdoc print the Rust version into the docs + // This replaces spaces with newlines because RUSTDOCFLAGS does not + // support arguments with regular spaces. Hopefully someday Cargo will + // have space support. + let rust_version = self.rust_version().replace(' ', "\n"); + rustdocflags.arg("--crate-version").arg(&rust_version); + + // Environment variables *required* throughout the build + // + // FIXME: should update code to not require this env var + cargo.env("CFG_COMPILER_HOST_TRIPLE", target.triple); + + // Set this for all builds to make sure doc builds also get it. + cargo.env("CFG_RELEASE_CHANNEL", &self.config.channel); + + // This one's a bit tricky. As of the time of this writing the compiler + // links to the `winapi` crate on crates.io. This crate provides raw + // bindings to Windows system functions, sort of like libc does for + // Unix. This crate also, however, provides "import libraries" for the + // MinGW targets. There's an import library per dll in the windows + // distribution which is what's linked to. These custom import libraries + // are used because the winapi crate can reference Windows functions not + // present in the MinGW import libraries. + // + // For example MinGW may ship libdbghelp.a, but it may not have + // references to all the functions in the dbghelp dll. Instead the + // custom import library for dbghelp in the winapi crates has all this + // information. + // + // Unfortunately for us though the import libraries are linked by + // default via `-ldylib=winapi_foo`. That is, they're linked with the + // `dylib` type with a `winapi_` prefix (so the winapi ones don't + // conflict with the system MinGW ones). This consequently means that + // the binaries we ship of things like rustc_codegen_llvm (aka the rustc_codegen_llvm + // DLL) when linked against *again*, for example with procedural macros + // or plugins, will trigger the propagation logic of `-ldylib`, passing + // `-lwinapi_foo` to the linker again. This isn't actually available in + // our distribution, however, so the link fails. + // + // To solve this problem we tell winapi to not use its bundled import + // libraries. This means that it will link to the system MinGW import + // libraries by default, and the `-ldylib=foo` directives will still get + // passed to the final linker, but they'll look like `-lfoo` which can + // be resolved because MinGW has the import library. The downside is we + // don't get newer functions from Windows, but we don't use any of them + // anyway. + if !mode.is_tool() { + cargo.env("WINAPI_NO_BUNDLED_LIBRARIES", "1"); + } + + for _ in 0..self.verbosity { + cargo.arg("-v"); + } + + match (mode, self.config.rust_codegen_units_std, self.config.rust_codegen_units) { + (Mode::Std, Some(n), _) | (_, _, Some(n)) => { + cargo.env(profile_var("CODEGEN_UNITS"), n.to_string()); + } + _ => { + // Don't set anything + } + } + + if self.config.rust_optimize { + // FIXME: cargo bench/install do not accept `--release` + if cmd != "bench" && cmd != "install" { + cargo.arg("--release"); + } + } + + if self.config.locked_deps { + cargo.arg("--locked"); + } + if self.config.vendor || self.is_sudo { + cargo.arg("--frozen"); + } + + // Try to use a sysroot-relative bindir, in case it was configured absolutely. + cargo.env("RUSTC_INSTALL_BINDIR", self.config.bindir_relative()); + + self.ci_env.force_coloring_in_ci(&mut cargo); + + // When we build Rust dylibs they're all intended for intermediate + // usage, so make sure we pass the -Cprefer-dynamic flag instead of + // linking all deps statically into the dylib. + if matches!(mode, Mode::Std | Mode::Rustc) { + rustflags.arg("-Cprefer-dynamic"); + } + + // When building incrementally we default to a lower ThinLTO import limit + // (unless explicitly specified otherwise). This will produce a somewhat + // slower code but give way better compile times. + { + let limit = match self.config.rust_thin_lto_import_instr_limit { + Some(limit) => Some(limit), + None if self.config.incremental => Some(10), + _ => None, + }; + + if let Some(limit) = limit { + rustflags.arg(&format!("-Cllvm-args=-import-instr-limit={}", limit)); + } + } + + Cargo { command: cargo, rustflags, rustdocflags } + } + + /// Ensure that a given step is built, returning its output. This will + /// cache the step, so it is safe (and good!) to call this as often as + /// needed to ensure that all dependencies are built. + pub fn ensure<S: Step>(&'a self, step: S) -> S::Output { + { + let mut stack = self.stack.borrow_mut(); + for stack_step in stack.iter() { + // should skip + if stack_step.downcast_ref::<S>().map_or(true, |stack_step| *stack_step != step) { + continue; + } + let mut out = String::new(); + out += &format!("\n\nCycle in build detected when adding {:?}\n", step); + for el in stack.iter().rev() { + out += &format!("\t{:?}\n", el); + } + panic!("{}", out); + } + if let Some(out) = self.cache.get(&step) { + self.verbose_than(1, &format!("{}c {:?}", " ".repeat(stack.len()), step)); + + return out; + } + self.verbose_than(1, &format!("{}> {:?}", " ".repeat(stack.len()), step)); + stack.push(Box::new(step.clone())); + } + + #[cfg(feature = "build-metrics")] + self.metrics.enter_step(&step); + + let (out, dur) = { + let start = Instant::now(); + let zero = Duration::new(0, 0); + let parent = self.time_spent_on_dependencies.replace(zero); + let out = step.clone().run(self); + let dur = start.elapsed(); + let deps = self.time_spent_on_dependencies.replace(parent + dur); + (out, dur - deps) + }; + + if self.config.print_step_timings && !self.config.dry_run { + let step_string = format!("{:?}", step); + let brace_index = step_string.find("{").unwrap_or(0); + let type_string = type_name::<S>(); + println!( + "[TIMING] {} {} -- {}.{:03}", + &type_string.strip_prefix("bootstrap::").unwrap_or(type_string), + &step_string[brace_index..], + dur.as_secs(), + dur.subsec_millis() + ); + } + + #[cfg(feature = "build-metrics")] + self.metrics.exit_step(); + + { + let mut stack = self.stack.borrow_mut(); + let cur_step = stack.pop().expect("step stack empty"); + assert_eq!(cur_step.downcast_ref(), Some(&step)); + } + self.verbose_than(1, &format!("{}< {:?}", " ".repeat(self.stack.borrow().len()), step)); + self.cache.put(step, out.clone()); + out + } + + /// Ensure that a given step is built *only if it's supposed to be built by default*, returning + /// its output. This will cache the step, so it's safe (and good!) to call this as often as + /// needed to ensure that all dependencies are build. + pub(crate) fn ensure_if_default<T, S: Step<Output = Option<T>>>( + &'a self, + step: S, + kind: Kind, + ) -> S::Output { + let desc = StepDescription::from::<S>(kind); + let should_run = (desc.should_run)(ShouldRun::new(self, desc.kind)); + + // Avoid running steps contained in --exclude + for pathset in &should_run.paths { + if desc.is_excluded(self, pathset) { + return None; + } + } + + // Only execute if it's supposed to run as default + if desc.default && should_run.is_really_default() { self.ensure(step) } else { None } + } + + /// Checks if any of the "should_run" paths is in the `Builder` paths. + pub(crate) fn was_invoked_explicitly<S: Step>(&'a self, kind: Kind) -> bool { + let desc = StepDescription::from::<S>(kind); + let should_run = (desc.should_run)(ShouldRun::new(self, desc.kind)); + + for path in &self.paths { + if should_run.paths.iter().any(|s| s.has(path, Some(desc.kind))) + && !desc.is_excluded( + self, + &PathSet::Suite(TaskPath { path: path.clone(), kind: Some(desc.kind) }), + ) + { + return true; + } + } + + false + } +} + +#[cfg(test)] +mod tests; + +#[derive(Debug, Clone)] +struct Rustflags(String, TargetSelection); + +impl Rustflags { + fn new(target: TargetSelection) -> Rustflags { + let mut ret = Rustflags(String::new(), target); + ret.propagate_cargo_env("RUSTFLAGS"); + ret + } + + /// By default, cargo will pick up on various variables in the environment. However, bootstrap + /// reuses those variables to pass additional flags to rustdoc, so by default they get overridden. + /// Explicitly add back any previous value in the environment. + /// + /// `prefix` is usually `RUSTFLAGS` or `RUSTDOCFLAGS`. + fn propagate_cargo_env(&mut self, prefix: &str) { + // Inherit `RUSTFLAGS` by default ... + self.env(prefix); + + // ... and also handle target-specific env RUSTFLAGS if they're configured. + let target_specific = format!("CARGO_TARGET_{}_{}", crate::envify(&self.1.triple), prefix); + self.env(&target_specific); + } + + fn env(&mut self, env: &str) { + if let Ok(s) = env::var(env) { + for part in s.split(' ') { + self.arg(part); + } + } + } + + fn arg(&mut self, arg: &str) -> &mut Self { + assert_eq!(arg.split(' ').count(), 1); + if !self.0.is_empty() { + self.0.push(' '); + } + self.0.push_str(arg); + self + } +} + +#[derive(Debug)] +pub struct Cargo { + command: Command, + rustflags: Rustflags, + rustdocflags: Rustflags, +} + +impl Cargo { + pub fn rustdocflag(&mut self, arg: &str) -> &mut Cargo { + self.rustdocflags.arg(arg); + self + } + pub fn rustflag(&mut self, arg: &str) -> &mut Cargo { + self.rustflags.arg(arg); + self + } + + pub fn arg(&mut self, arg: impl AsRef<OsStr>) -> &mut Cargo { + self.command.arg(arg.as_ref()); + self + } + + pub fn args<I, S>(&mut self, args: I) -> &mut Cargo + where + I: IntoIterator<Item = S>, + S: AsRef<OsStr>, + { + for arg in args { + self.arg(arg.as_ref()); + } + self + } + + pub fn env(&mut self, key: impl AsRef<OsStr>, value: impl AsRef<OsStr>) -> &mut Cargo { + // These are managed through rustflag/rustdocflag interfaces. + assert_ne!(key.as_ref(), "RUSTFLAGS"); + assert_ne!(key.as_ref(), "RUSTDOCFLAGS"); + self.command.env(key.as_ref(), value.as_ref()); + self + } + + pub fn add_rustc_lib_path(&mut self, builder: &Builder<'_>, compiler: Compiler) { + builder.add_rustc_lib_path(compiler, &mut self.command); + } + + pub fn current_dir(&mut self, dir: &Path) -> &mut Cargo { + self.command.current_dir(dir); + self + } +} + +impl From<Cargo> for Command { + fn from(mut cargo: Cargo) -> Command { + let rustflags = &cargo.rustflags.0; + if !rustflags.is_empty() { + cargo.command.env("RUSTFLAGS", rustflags); + } + + let rustdocflags = &cargo.rustdocflags.0; + if !rustdocflags.is_empty() { + cargo.command.env("RUSTDOCFLAGS", rustdocflags); + } + + cargo.command + } +} diff --git a/src/bootstrap/builder/tests.rs b/src/bootstrap/builder/tests.rs new file mode 100644 index 000000000..c084e77d3 --- /dev/null +++ b/src/bootstrap/builder/tests.rs @@ -0,0 +1,671 @@ +use super::*; +use crate::config::{Config, TargetSelection}; +use std::thread; + +fn configure(cmd: &str, host: &[&str], target: &[&str]) -> Config { + configure_with_args(&[cmd.to_owned()], host, target) +} + +fn configure_with_args(cmd: &[String], host: &[&str], target: &[&str]) -> Config { + let mut config = Config::parse(cmd); + // don't save toolstates + config.save_toolstates = None; + config.dry_run = true; + + // Ignore most submodules, since we don't need them for a dry run. + // But make sure to check out the `doc` and `rust-analyzer` submodules, since some steps need them + // just to know which commands to run. + let submodule_build = Build::new(Config { + // don't include LLVM, so CI doesn't require ninja/cmake to be installed + rust_codegen_backends: vec![], + ..Config::parse(&["check".to_owned()]) + }); + submodule_build.update_submodule(Path::new("src/doc/book")); + submodule_build.update_submodule(Path::new("src/tools/rust-analyzer")); + config.submodules = Some(false); + + config.ninja_in_file = false; + // try to avoid spurious failures in dist where we create/delete each others file + // HACK: rather than pull in `tempdir`, use the one that cargo has conveniently created for us + let dir = Path::new(env!("OUT_DIR")) + .join("tmp-rustbuild-tests") + .join(&thread::current().name().unwrap_or("unknown").replace(":", "-")); + t!(fs::create_dir_all(&dir)); + config.out = dir; + config.build = TargetSelection::from_user("A"); + config.hosts = host.iter().map(|s| TargetSelection::from_user(s)).collect(); + config.targets = target.iter().map(|s| TargetSelection::from_user(s)).collect(); + config +} + +fn first<A, B>(v: Vec<(A, B)>) -> Vec<A> { + v.into_iter().map(|(a, _)| a).collect::<Vec<_>>() +} + +fn run_build(paths: &[PathBuf], config: Config) -> Cache { + let kind = config.cmd.kind(); + let build = Build::new(config); + let builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(kind), paths); + builder.cache +} + +fn check_cli<const N: usize>(paths: [&str; N]) { + run_build( + &paths.map(PathBuf::from), + configure_with_args(&paths.map(String::from), &["A"], &["A"]), + ); +} + +macro_rules! std { + ($host:ident => $target:ident, stage = $stage:literal) => { + compile::Std::new( + Compiler { host: TargetSelection::from_user(stringify!($host)), stage: $stage }, + TargetSelection::from_user(stringify!($target)), + ) + }; +} + +macro_rules! rustc { + ($host:ident => $target:ident, stage = $stage:literal) => { + compile::Rustc::new( + Compiler { host: TargetSelection::from_user(stringify!($host)), stage: $stage }, + TargetSelection::from_user(stringify!($target)), + ) + }; +} + +#[test] +fn test_valid() { + // make sure multi suite paths are accepted + check_cli(["test", "src/test/ui/attr-start.rs", "src/test/ui/attr-shebang.rs"]); +} + +#[test] +#[should_panic] +fn test_invalid() { + // make sure that invalid paths are caught, even when combined with valid paths + check_cli(["test", "library/std", "x"]); +} + +#[test] +fn test_intersection() { + let set = PathSet::Set( + ["library/core", "library/alloc", "library/std"].into_iter().map(TaskPath::parse).collect(), + ); + let mut command_paths = + vec![Path::new("library/core"), Path::new("library/alloc"), Path::new("library/stdarch")]; + let subset = set.intersection_removing_matches(&mut command_paths, None); + assert_eq!( + subset, + PathSet::Set(["library/core", "library/alloc"].into_iter().map(TaskPath::parse).collect()) + ); + assert_eq!(command_paths, vec![Path::new("library/stdarch")]); +} + +#[test] +fn test_exclude() { + let mut config = configure("test", &["A"], &["A"]); + config.exclude = vec![TaskPath::parse("src/tools/tidy")]; + let cache = run_build(&[], config); + + // Ensure we have really excluded tidy + assert!(!cache.contains::<test::Tidy>()); + + // Ensure other tests are not affected. + assert!(cache.contains::<test::RustdocUi>()); +} + +#[test] +fn test_exclude_kind() { + let path = PathBuf::from("src/tools/cargotest"); + let exclude = TaskPath::parse("test::src/tools/cargotest"); + assert_eq!(exclude, TaskPath { kind: Some(Kind::Test), path: path.clone() }); + + let mut config = configure("test", &["A"], &["A"]); + // Ensure our test is valid, and `test::Cargotest` would be run without the exclude. + assert!(run_build(&[path.clone()], config.clone()).contains::<test::Cargotest>()); + // Ensure tests for cargotest are skipped. + config.exclude = vec![exclude.clone()]; + assert!(!run_build(&[path.clone()], config).contains::<test::Cargotest>()); + + // Ensure builds for cargotest are not skipped. + let mut config = configure("build", &["A"], &["A"]); + config.exclude = vec![exclude]; + assert!(run_build(&[path], config).contains::<tool::CargoTest>()); +} + +/// Ensure that if someone passes both a single crate and `library`, all library crates get built. +#[test] +fn alias_and_path_for_library() { + let mut cache = + run_build(&["library".into(), "core".into()], configure("build", &["A"], &["A"])); + assert_eq!( + first(cache.all::<compile::Std>()), + &[std!(A => A, stage = 0), std!(A => A, stage = 1)] + ); +} + +mod defaults { + use super::{configure, first, run_build}; + use crate::builder::*; + use crate::Config; + use pretty_assertions::assert_eq; + + #[test] + fn build_default() { + let mut cache = run_build(&[], configure("build", &["A"], &["A"])); + + let a = TargetSelection::from_user("A"); + assert_eq!( + first(cache.all::<compile::Std>()), + &[std!(A => A, stage = 0), std!(A => A, stage = 1),] + ); + assert!(!cache.all::<compile::Assemble>().is_empty()); + // Make sure rustdoc is only built once. + assert_eq!( + first(cache.all::<tool::Rustdoc>()), + // Recall that rustdoc stages are off-by-one + // - this is the compiler it's _linked_ to, not built with. + &[tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }], + ); + assert_eq!(first(cache.all::<compile::Rustc>()), &[rustc!(A => A, stage = 0)],); + } + + #[test] + fn build_stage_0() { + let config = Config { stage: 0, ..configure("build", &["A"], &["A"]) }; + let mut cache = run_build(&[], config); + + let a = TargetSelection::from_user("A"); + assert_eq!(first(cache.all::<compile::Std>()), &[std!(A => A, stage = 0)]); + assert!(!cache.all::<compile::Assemble>().is_empty()); + assert_eq!( + first(cache.all::<tool::Rustdoc>()), + // This is the beta rustdoc. + // Add an assert here to make sure this is the only rustdoc built. + &[tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } }], + ); + assert!(cache.all::<compile::Rustc>().is_empty()); + } + + #[test] + fn build_cross_compile() { + let config = Config { stage: 1, ..configure("build", &["A", "B"], &["A", "B"]) }; + let mut cache = run_build(&[], config); + + let a = TargetSelection::from_user("A"); + let b = TargetSelection::from_user("B"); + + // Ideally, this build wouldn't actually have `target: a` + // rustdoc/rustcc/std here (the user only requested a host=B build, so + // there's not really a need for us to build for target A in this case + // (since we're producing stage 1 libraries/binaries). But currently + // rustbuild is just a bit buggy here; this should be fixed though. + assert_eq!( + first(cache.all::<compile::Std>()), + &[ + std!(A => A, stage = 0), + std!(A => A, stage = 1), + std!(A => B, stage = 0), + std!(A => B, stage = 1), + ] + ); + assert_eq!( + first(cache.all::<compile::Assemble>()), + &[ + compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, + compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, + compile::Assemble { target_compiler: Compiler { host: b, stage: 1 } }, + ] + ); + assert_eq!( + first(cache.all::<tool::Rustdoc>()), + &[ + tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }, + tool::Rustdoc { compiler: Compiler { host: b, stage: 1 } }, + ], + ); + assert_eq!( + first(cache.all::<compile::Rustc>()), + &[rustc!(A => A, stage = 0), rustc!(A => B, stage = 0),] + ); + } + + #[test] + fn doc_default() { + let mut config = configure("doc", &["A"], &["A"]); + config.compiler_docs = true; + config.cmd = Subcommand::Doc { paths: Vec::new(), open: false }; + let mut cache = run_build(&[], config); + let a = TargetSelection::from_user("A"); + + // error_index_generator uses stage 0 to share rustdoc artifacts with the + // rustdoc tool. + assert_eq!(first(cache.all::<doc::ErrorIndex>()), &[doc::ErrorIndex { target: a },]); + assert_eq!( + first(cache.all::<tool::ErrorIndex>()), + &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 0 } }] + ); + // docs should be built with the beta compiler, not with the stage0 artifacts. + // recall that rustdoc is off-by-one: `stage` is the compiler rustdoc is _linked_ to, + // not the one it was built by. + assert_eq!( + first(cache.all::<tool::Rustdoc>()), + &[tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } },] + ); + } +} + +mod dist { + use super::{first, run_build, Config}; + use crate::builder::*; + use pretty_assertions::assert_eq; + + fn configure(host: &[&str], target: &[&str]) -> Config { + Config { stage: 2, ..super::configure("dist", host, target) } + } + + #[test] + fn dist_baseline() { + let mut cache = run_build(&[], configure(&["A"], &["A"])); + + let a = TargetSelection::from_user("A"); + + assert_eq!(first(cache.all::<dist::Docs>()), &[dist::Docs { host: a },]); + assert_eq!(first(cache.all::<dist::Mingw>()), &[dist::Mingw { host: a },]); + assert_eq!( + first(cache.all::<dist::Rustc>()), + &[dist::Rustc { compiler: Compiler { host: a, stage: 2 } },] + ); + assert_eq!( + first(cache.all::<dist::Std>()), + &[dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a },] + ); + assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]); + // Make sure rustdoc is only built once. + assert_eq!( + first(cache.all::<tool::Rustdoc>()), + &[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },] + ); + } + + #[test] + fn dist_with_targets() { + let mut cache = run_build(&[], configure(&["A"], &["A", "B"])); + + let a = TargetSelection::from_user("A"); + let b = TargetSelection::from_user("B"); + + assert_eq!( + first(cache.all::<dist::Docs>()), + &[dist::Docs { host: a }, dist::Docs { host: b },] + ); + assert_eq!( + first(cache.all::<dist::Mingw>()), + &[dist::Mingw { host: a }, dist::Mingw { host: b },] + ); + assert_eq!( + first(cache.all::<dist::Rustc>()), + &[dist::Rustc { compiler: Compiler { host: a, stage: 2 } },] + ); + assert_eq!( + first(cache.all::<dist::Std>()), + &[ + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, + dist::Std { compiler: Compiler { host: a, stage: 2 }, target: b }, + ] + ); + assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]); + } + + #[test] + fn dist_with_hosts() { + let mut cache = run_build(&[], configure(&["A", "B"], &["A", "B"])); + + let a = TargetSelection::from_user("A"); + let b = TargetSelection::from_user("B"); + + assert_eq!( + first(cache.all::<dist::Docs>()), + &[dist::Docs { host: a }, dist::Docs { host: b },] + ); + assert_eq!( + first(cache.all::<dist::Mingw>()), + &[dist::Mingw { host: a }, dist::Mingw { host: b },] + ); + assert_eq!( + first(cache.all::<dist::Rustc>()), + &[ + dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, + dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, + ] + ); + assert_eq!( + first(cache.all::<dist::Std>()), + &[ + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, + ] + ); + assert_eq!( + first(cache.all::<compile::Std>()), + &[ + std!(A => A, stage = 0), + std!(A => A, stage = 1), + std!(A => A, stage = 2), + std!(A => B, stage = 1), + std!(A => B, stage = 2), + ], + ); + assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]); + } + + #[test] + fn dist_only_cross_host() { + let b = TargetSelection::from_user("B"); + let mut config = configure(&["A", "B"], &["A", "B"]); + config.docs = false; + config.extended = true; + config.hosts = vec![b]; + let mut cache = run_build(&[], config); + + assert_eq!( + first(cache.all::<dist::Rustc>()), + &[dist::Rustc { compiler: Compiler { host: b, stage: 2 } },] + ); + assert_eq!( + first(cache.all::<compile::Rustc>()), + &[rustc!(A => A, stage = 0), rustc!(A => B, stage = 1),] + ); + } + + #[test] + fn dist_with_targets_and_hosts() { + let mut cache = run_build(&[], configure(&["A", "B"], &["A", "B", "C"])); + + let a = TargetSelection::from_user("A"); + let b = TargetSelection::from_user("B"); + let c = TargetSelection::from_user("C"); + + assert_eq!( + first(cache.all::<dist::Docs>()), + &[dist::Docs { host: a }, dist::Docs { host: b }, dist::Docs { host: c },] + ); + assert_eq!( + first(cache.all::<dist::Mingw>()), + &[dist::Mingw { host: a }, dist::Mingw { host: b }, dist::Mingw { host: c },] + ); + assert_eq!( + first(cache.all::<dist::Rustc>()), + &[ + dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, + dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, + ] + ); + assert_eq!( + first(cache.all::<dist::Std>()), + &[ + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, + dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c }, + ] + ); + assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]); + } + + #[test] + fn dist_with_empty_host() { + let config = configure(&[], &["C"]); + let mut cache = run_build(&[], config); + + let a = TargetSelection::from_user("A"); + let c = TargetSelection::from_user("C"); + + assert_eq!(first(cache.all::<dist::Docs>()), &[dist::Docs { host: c },]); + assert_eq!(first(cache.all::<dist::Mingw>()), &[dist::Mingw { host: c },]); + assert_eq!( + first(cache.all::<dist::Std>()), + &[dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c },] + ); + } + + #[test] + fn dist_with_same_targets_and_hosts() { + let mut cache = run_build(&[], configure(&["A", "B"], &["A", "B"])); + + let a = TargetSelection::from_user("A"); + let b = TargetSelection::from_user("B"); + + assert_eq!( + first(cache.all::<dist::Docs>()), + &[dist::Docs { host: a }, dist::Docs { host: b },] + ); + assert_eq!( + first(cache.all::<dist::Mingw>()), + &[dist::Mingw { host: a }, dist::Mingw { host: b },] + ); + assert_eq!( + first(cache.all::<dist::Rustc>()), + &[ + dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, + dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, + ] + ); + assert_eq!( + first(cache.all::<dist::Std>()), + &[ + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, + ] + ); + assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]); + assert_eq!( + first(cache.all::<compile::Std>()), + &[ + std!(A => A, stage = 0), + std!(A => A, stage = 1), + std!(A => A, stage = 2), + std!(A => B, stage = 1), + std!(A => B, stage = 2), + ] + ); + assert_eq!( + first(cache.all::<compile::Assemble>()), + &[ + compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, + compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, + compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } }, + compile::Assemble { target_compiler: Compiler { host: b, stage: 2 } }, + ] + ); + } + + #[test] + fn build_all() { + let build = Build::new(configure(&["A", "B"], &["A", "B", "C"])); + let mut builder = Builder::new(&build); + builder.run_step_descriptions( + &Builder::get_step_descriptions(Kind::Build), + &["compiler/rustc".into(), "library".into()], + ); + + assert_eq!( + first(builder.cache.all::<compile::Std>()), + &[ + std!(A => A, stage = 0), + std!(A => A, stage = 1), + std!(A => A, stage = 2), + std!(A => B, stage = 1), + std!(A => B, stage = 2), + std!(A => C, stage = 2), + ] + ); + assert_eq!(builder.cache.all::<compile::Assemble>().len(), 5); + assert_eq!( + first(builder.cache.all::<compile::Rustc>()), + &[ + rustc!(A => A, stage = 0), + rustc!(A => A, stage = 1), + rustc!(A => A, stage = 2), + rustc!(A => B, stage = 1), + rustc!(A => B, stage = 2), + ] + ); + } + + #[test] + fn build_with_empty_host() { + let config = configure(&[], &["C"]); + let build = Build::new(config); + let mut builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]); + + let a = TargetSelection::from_user("A"); + + assert_eq!( + first(builder.cache.all::<compile::Std>()), + &[std!(A => A, stage = 0), std!(A => A, stage = 1), std!(A => C, stage = 2),] + ); + assert_eq!( + first(builder.cache.all::<compile::Assemble>()), + &[ + compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, + compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, + compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } }, + ] + ); + assert_eq!( + first(builder.cache.all::<compile::Rustc>()), + &[rustc!(A => A, stage = 0), rustc!(A => A, stage = 1),] + ); + } + + #[test] + fn test_with_no_doc_stage0() { + let mut config = configure(&["A"], &["A"]); + config.stage = 0; + config.cmd = Subcommand::Test { + paths: vec!["library/std".into()], + skip: vec![], + test_args: vec![], + rustc_args: vec![], + fail_fast: true, + doc_tests: DocTests::No, + bless: false, + force_rerun: false, + compare_mode: None, + rustfix_coverage: false, + pass: None, + run: None, + }; + + let build = Build::new(config); + let mut builder = Builder::new(&build); + + let host = TargetSelection::from_user("A"); + + builder.run_step_descriptions( + &[StepDescription::from::<test::Crate>(Kind::Test)], + &["library/std".into()], + ); + + // Ensure we don't build any compiler artifacts. + assert!(!builder.cache.contains::<compile::Rustc>()); + assert_eq!( + first(builder.cache.all::<test::Crate>()), + &[test::Crate { + compiler: Compiler { host, stage: 0 }, + target: host, + mode: Mode::Std, + test_kind: test::TestKind::Test, + crates: vec![INTERNER.intern_str("std")], + },] + ); + } + + #[test] + fn doc_ci() { + let mut config = configure(&["A"], &["A"]); + config.compiler_docs = true; + config.cmd = Subcommand::Doc { paths: Vec::new(), open: false }; + let build = Build::new(config); + let mut builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Doc), &[]); + let a = TargetSelection::from_user("A"); + + // error_index_generator uses stage 1 to share rustdoc artifacts with the + // rustdoc tool. + assert_eq!( + first(builder.cache.all::<doc::ErrorIndex>()), + &[doc::ErrorIndex { target: a },] + ); + assert_eq!( + first(builder.cache.all::<tool::ErrorIndex>()), + &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 1 } }] + ); + // This is actually stage 1, but Rustdoc::run swaps out the compiler with + // stage minus 1 if --stage is not 0. Very confusing! + assert_eq!( + first(builder.cache.all::<tool::Rustdoc>()), + &[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },] + ); + } + + #[test] + fn test_docs() { + // Behavior of `x.py test` doing various documentation tests. + let mut config = configure(&["A"], &["A"]); + config.cmd = Subcommand::Test { + paths: vec![], + skip: vec![], + test_args: vec![], + rustc_args: vec![], + fail_fast: true, + doc_tests: DocTests::Yes, + bless: false, + force_rerun: false, + compare_mode: None, + rustfix_coverage: false, + pass: None, + run: None, + }; + // Make sure rustfmt binary not being found isn't an error. + config.channel = "beta".to_string(); + let build = Build::new(config); + let mut builder = Builder::new(&build); + + builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Test), &[]); + let a = TargetSelection::from_user("A"); + + // error_index_generator uses stage 1 to share rustdoc artifacts with the + // rustdoc tool. + assert_eq!( + first(builder.cache.all::<doc::ErrorIndex>()), + &[doc::ErrorIndex { target: a },] + ); + assert_eq!( + first(builder.cache.all::<tool::ErrorIndex>()), + &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 1 } }] + ); + // Unfortunately rustdoc is built twice. Once from stage1 for compiletest + // (and other things), and once from stage0 for std crates. Ideally it + // would only be built once. If someone wants to fix this, it might be + // worth investigating if it would be possible to test std from stage1. + // Note that the stages here are +1 than what they actually are because + // Rustdoc::run swaps out the compiler with stage minus 1 if --stage is + // not 0. + // + // The stage 0 copy is the one downloaded for bootstrapping. It is + // (currently) needed to run "cargo test" on the linkchecker, and + // should be relatively "free". + assert_eq!( + first(builder.cache.all::<tool::Rustdoc>()), + &[ + tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } }, + tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }, + tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } }, + ] + ); + } +} diff --git a/src/bootstrap/cache.rs b/src/bootstrap/cache.rs new file mode 100644 index 000000000..be5c9bb07 --- /dev/null +++ b/src/bootstrap/cache.rs @@ -0,0 +1,273 @@ +use std::any::{Any, TypeId}; +use std::borrow::Borrow; +use std::cell::RefCell; +use std::cmp::{Ord, Ordering, PartialOrd}; +use std::collections::HashMap; +use std::convert::AsRef; +use std::fmt; +use std::hash::{Hash, Hasher}; +use std::marker::PhantomData; +use std::mem; +use std::ops::Deref; +use std::path::PathBuf; +use std::sync::Mutex; + +// FIXME: replace with std::lazy after it gets stabilized and reaches beta +use once_cell::sync::Lazy; + +use crate::builder::Step; + +pub struct Interned<T>(usize, PhantomData<*const T>); + +impl<T: Internable + Default> Default for Interned<T> { + fn default() -> Self { + T::default().intern() + } +} + +impl<T> Copy for Interned<T> {} +impl<T> Clone for Interned<T> { + fn clone(&self) -> Interned<T> { + *self + } +} + +impl<T> PartialEq for Interned<T> { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } +} +impl<T> Eq for Interned<T> {} + +impl PartialEq<str> for Interned<String> { + fn eq(&self, other: &str) -> bool { + *self == other + } +} +impl<'a> PartialEq<&'a str> for Interned<String> { + fn eq(&self, other: &&str) -> bool { + **self == **other + } +} +impl<'a, T> PartialEq<&'a Interned<T>> for Interned<T> { + fn eq(&self, other: &&Self) -> bool { + self.0 == other.0 + } +} +impl<'a, T> PartialEq<Interned<T>> for &'a Interned<T> { + fn eq(&self, other: &Interned<T>) -> bool { + self.0 == other.0 + } +} + +unsafe impl<T> Send for Interned<T> {} +unsafe impl<T> Sync for Interned<T> {} + +impl fmt::Display for Interned<String> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s: &str = &*self; + f.write_str(s) + } +} + +impl<T, U: ?Sized + fmt::Debug> fmt::Debug for Interned<T> +where + Self: Deref<Target = U>, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s: &U = &*self; + f.write_fmt(format_args!("{:?}", s)) + } +} + +impl<T: Internable + Hash> Hash for Interned<T> { + fn hash<H: Hasher>(&self, state: &mut H) { + let l = T::intern_cache().lock().unwrap(); + l.get(*self).hash(state) + } +} + +impl<T: Internable + Deref> Deref for Interned<T> { + type Target = T::Target; + fn deref(&self) -> &'static Self::Target { + let l = T::intern_cache().lock().unwrap(); + unsafe { mem::transmute::<&Self::Target, &'static Self::Target>(l.get(*self)) } + } +} + +impl<T: Internable + AsRef<U>, U: ?Sized> AsRef<U> for Interned<T> { + fn as_ref(&self) -> &'static U { + let l = T::intern_cache().lock().unwrap(); + unsafe { mem::transmute::<&U, &'static U>(l.get(*self).as_ref()) } + } +} + +impl<T: Internable + PartialOrd> PartialOrd for Interned<T> { + fn partial_cmp(&self, other: &Self) -> Option<Ordering> { + let l = T::intern_cache().lock().unwrap(); + l.get(*self).partial_cmp(l.get(*other)) + } +} + +impl<T: Internable + Ord> Ord for Interned<T> { + fn cmp(&self, other: &Self) -> Ordering { + let l = T::intern_cache().lock().unwrap(); + l.get(*self).cmp(l.get(*other)) + } +} + +struct TyIntern<T: Clone + Eq> { + items: Vec<T>, + set: HashMap<T, Interned<T>>, +} + +impl<T: Hash + Clone + Eq> Default for TyIntern<T> { + fn default() -> Self { + TyIntern { items: Vec::new(), set: Default::default() } + } +} + +impl<T: Hash + Clone + Eq> TyIntern<T> { + fn intern_borrow<B>(&mut self, item: &B) -> Interned<T> + where + B: Eq + Hash + ToOwned<Owned = T> + ?Sized, + T: Borrow<B>, + { + if let Some(i) = self.set.get(&item) { + return *i; + } + let item = item.to_owned(); + let interned = Interned(self.items.len(), PhantomData::<*const T>); + self.set.insert(item.clone(), interned); + self.items.push(item); + interned + } + + fn intern(&mut self, item: T) -> Interned<T> { + if let Some(i) = self.set.get(&item) { + return *i; + } + let interned = Interned(self.items.len(), PhantomData::<*const T>); + self.set.insert(item.clone(), interned); + self.items.push(item); + interned + } + + fn get(&self, i: Interned<T>) -> &T { + &self.items[i.0] + } +} + +#[derive(Default)] +pub struct Interner { + strs: Mutex<TyIntern<String>>, + paths: Mutex<TyIntern<PathBuf>>, + lists: Mutex<TyIntern<Vec<String>>>, +} + +trait Internable: Clone + Eq + Hash + 'static { + fn intern_cache() -> &'static Mutex<TyIntern<Self>>; + + fn intern(self) -> Interned<Self> { + Self::intern_cache().lock().unwrap().intern(self) + } +} + +impl Internable for String { + fn intern_cache() -> &'static Mutex<TyIntern<Self>> { + &INTERNER.strs + } +} + +impl Internable for PathBuf { + fn intern_cache() -> &'static Mutex<TyIntern<Self>> { + &INTERNER.paths + } +} + +impl Internable for Vec<String> { + fn intern_cache() -> &'static Mutex<TyIntern<Self>> { + &INTERNER.lists + } +} + +impl Interner { + pub fn intern_str(&self, s: &str) -> Interned<String> { + self.strs.lock().unwrap().intern_borrow(s) + } + pub fn intern_string(&self, s: String) -> Interned<String> { + self.strs.lock().unwrap().intern(s) + } + + pub fn intern_path(&self, s: PathBuf) -> Interned<PathBuf> { + self.paths.lock().unwrap().intern(s) + } + + pub fn intern_list(&self, v: Vec<String>) -> Interned<Vec<String>> { + self.lists.lock().unwrap().intern(v) + } +} + +pub static INTERNER: Lazy<Interner> = Lazy::new(Interner::default); + +/// This is essentially a `HashMap` which allows storing any type in its input and +/// any type in its output. It is a write-once cache; values are never evicted, +/// which means that references to the value can safely be returned from the +/// `get()` method. +#[derive(Debug)] +pub struct Cache( + RefCell< + HashMap< + TypeId, + Box<dyn Any>, // actually a HashMap<Step, Interned<Step::Output>> + >, + >, +); + +impl Cache { + pub fn new() -> Cache { + Cache(RefCell::new(HashMap::new())) + } + + pub fn put<S: Step>(&self, step: S, value: S::Output) { + let mut cache = self.0.borrow_mut(); + let type_id = TypeId::of::<S>(); + let stepcache = cache + .entry(type_id) + .or_insert_with(|| Box::new(HashMap::<S, S::Output>::new())) + .downcast_mut::<HashMap<S, S::Output>>() + .expect("invalid type mapped"); + assert!(!stepcache.contains_key(&step), "processing {:?} a second time", step); + stepcache.insert(step, value); + } + + pub fn get<S: Step>(&self, step: &S) -> Option<S::Output> { + let mut cache = self.0.borrow_mut(); + let type_id = TypeId::of::<S>(); + let stepcache = cache + .entry(type_id) + .or_insert_with(|| Box::new(HashMap::<S, S::Output>::new())) + .downcast_mut::<HashMap<S, S::Output>>() + .expect("invalid type mapped"); + stepcache.get(step).cloned() + } +} + +#[cfg(test)] +impl Cache { + pub fn all<S: Ord + Clone + Step>(&mut self) -> Vec<(S, S::Output)> { + let cache = self.0.get_mut(); + let type_id = TypeId::of::<S>(); + let mut v = cache + .remove(&type_id) + .map(|b| b.downcast::<HashMap<S, S::Output>>().expect("correct type")) + .map(|m| m.into_iter().collect::<Vec<_>>()) + .unwrap_or_default(); + v.sort_by_key(|(s, _)| s.clone()); + v + } + + pub fn contains<S: Step>(&self) -> bool { + self.0.borrow().contains_key(&TypeId::of::<S>()) + } +} diff --git a/src/bootstrap/cc_detect.rs b/src/bootstrap/cc_detect.rs new file mode 100644 index 000000000..759a99c33 --- /dev/null +++ b/src/bootstrap/cc_detect.rs @@ -0,0 +1,252 @@ +//! C-compiler probing and detection. +//! +//! This module will fill out the `cc` and `cxx` maps of `Build` by looking for +//! C and C++ compilers for each target configured. A compiler is found through +//! a number of vectors (in order of precedence) +//! +//! 1. Configuration via `target.$target.cc` in `config.toml`. +//! 2. Configuration via `target.$target.android-ndk` in `config.toml`, if +//! applicable +//! 3. Special logic to probe on OpenBSD +//! 4. The `CC_$target` environment variable. +//! 5. The `CC` environment variable. +//! 6. "cc" +//! +//! Some of this logic is implemented here, but much of it is farmed out to the +//! `cc` crate itself, so we end up having the same fallbacks as there. +//! Similar logic is then used to find a C++ compiler, just some s/cc/c++/ is +//! used. +//! +//! It is intended that after this module has run no C/C++ compiler will +//! ever be probed for. Instead the compilers found here will be used for +//! everything. + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::{env, iter}; + +use crate::config::{Target, TargetSelection}; +use crate::util::output; +use crate::{Build, CLang, GitRepo}; + +// The `cc` crate doesn't provide a way to obtain a path to the detected archiver, +// so use some simplified logic here. First we respect the environment variable `AR`, then +// try to infer the archiver path from the C compiler path. +// In the future this logic should be replaced by calling into the `cc` crate. +fn cc2ar(cc: &Path, target: TargetSelection) -> Option<PathBuf> { + if let Some(ar) = env::var_os(format!("AR_{}", target.triple.replace("-", "_"))) { + Some(PathBuf::from(ar)) + } else if let Some(ar) = env::var_os("AR") { + Some(PathBuf::from(ar)) + } else if target.contains("msvc") { + None + } else if target.contains("musl") { + Some(PathBuf::from("ar")) + } else if target.contains("openbsd") { + Some(PathBuf::from("ar")) + } else if target.contains("vxworks") { + Some(PathBuf::from("wr-ar")) + } else { + let parent = cc.parent().unwrap(); + let file = cc.file_name().unwrap().to_str().unwrap(); + for suffix in &["gcc", "cc", "clang"] { + if let Some(idx) = file.rfind(suffix) { + let mut file = file[..idx].to_owned(); + file.push_str("ar"); + return Some(parent.join(&file)); + } + } + Some(parent.join(file)) + } +} + +fn new_cc_build(build: &Build, target: TargetSelection) -> cc::Build { + let mut cfg = cc::Build::new(); + cfg.cargo_metadata(false) + .opt_level(2) + .warnings(false) + .debug(false) + .target(&target.triple) + .host(&build.build.triple); + match build.crt_static(target) { + Some(a) => { + cfg.static_crt(a); + } + None => { + if target.contains("msvc") { + cfg.static_crt(true); + } + if target.contains("musl") { + cfg.static_flag(true); + } + } + } + cfg +} + +pub fn find(build: &mut Build) { + // For all targets we're going to need a C compiler for building some shims + // and such as well as for being a linker for Rust code. + let targets = build + .targets + .iter() + .chain(&build.hosts) + .cloned() + .chain(iter::once(build.build)) + .collect::<HashSet<_>>(); + for target in targets.into_iter() { + let mut cfg = new_cc_build(build, target); + let config = build.config.target_config.get(&target); + if let Some(cc) = config.and_then(|c| c.cc.as_ref()) { + cfg.compiler(cc); + } else { + set_compiler(&mut cfg, Language::C, target, config, build); + } + + let compiler = cfg.get_compiler(); + let ar = if let ar @ Some(..) = config.and_then(|c| c.ar.clone()) { + ar + } else { + cc2ar(compiler.path(), target) + }; + + build.cc.insert(target, compiler.clone()); + let cflags = build.cflags(target, GitRepo::Rustc, CLang::C); + + // If we use llvm-libunwind, we will need a C++ compiler as well for all targets + // We'll need one anyways if the target triple is also a host triple + let mut cfg = new_cc_build(build, target); + cfg.cpp(true); + let cxx_configured = if let Some(cxx) = config.and_then(|c| c.cxx.as_ref()) { + cfg.compiler(cxx); + true + } else if build.hosts.contains(&target) || build.build == target { + set_compiler(&mut cfg, Language::CPlusPlus, target, config, build); + true + } else { + // Use an auto-detected compiler (or one configured via `CXX_target_triple` env vars). + cfg.try_get_compiler().is_ok() + }; + + // for VxWorks, record CXX compiler which will be used in lib.rs:linker() + if cxx_configured || target.contains("vxworks") { + let compiler = cfg.get_compiler(); + build.cxx.insert(target, compiler); + } + + build.verbose(&format!("CC_{} = {:?}", &target.triple, build.cc(target))); + build.verbose(&format!("CFLAGS_{} = {:?}", &target.triple, cflags)); + if let Ok(cxx) = build.cxx(target) { + let cxxflags = build.cflags(target, GitRepo::Rustc, CLang::Cxx); + build.verbose(&format!("CXX_{} = {:?}", &target.triple, cxx)); + build.verbose(&format!("CXXFLAGS_{} = {:?}", &target.triple, cxxflags)); + } + if let Some(ar) = ar { + build.verbose(&format!("AR_{} = {:?}", &target.triple, ar)); + build.ar.insert(target, ar); + } + + if let Some(ranlib) = config.and_then(|c| c.ranlib.clone()) { + build.ranlib.insert(target, ranlib); + } + } +} + +fn set_compiler( + cfg: &mut cc::Build, + compiler: Language, + target: TargetSelection, + config: Option<&Target>, + build: &Build, +) { + match &*target.triple { + // When compiling for android we may have the NDK configured in the + // config.toml in which case we look there. Otherwise the default + // compiler already takes into account the triple in question. + t if t.contains("android") => { + if let Some(ndk) = config.and_then(|c| c.ndk.as_ref()) { + let target = target + .triple + .replace("armv7neon", "arm") + .replace("armv7", "arm") + .replace("thumbv7neon", "arm") + .replace("thumbv7", "arm"); + let compiler = format!("{}-{}", target, compiler.clang()); + cfg.compiler(ndk.join("bin").join(compiler)); + } + } + + // The default gcc version from OpenBSD may be too old, try using egcc, + // which is a gcc version from ports, if this is the case. + t if t.contains("openbsd") => { + let c = cfg.get_compiler(); + let gnu_compiler = compiler.gcc(); + if !c.path().ends_with(gnu_compiler) { + return; + } + + let output = output(c.to_command().arg("--version")); + let i = match output.find(" 4.") { + Some(i) => i, + None => return, + }; + match output[i + 3..].chars().next().unwrap() { + '0'..='6' => {} + _ => return, + } + let alternative = format!("e{}", gnu_compiler); + if Command::new(&alternative).output().is_ok() { + cfg.compiler(alternative); + } + } + + "mips-unknown-linux-musl" => { + if cfg.get_compiler().path().to_str() == Some("gcc") { + cfg.compiler("mips-linux-musl-gcc"); + } + } + "mipsel-unknown-linux-musl" => { + if cfg.get_compiler().path().to_str() == Some("gcc") { + cfg.compiler("mipsel-linux-musl-gcc"); + } + } + + t if t.contains("musl") => { + if let Some(root) = build.musl_root(target) { + let guess = root.join("bin/musl-gcc"); + if guess.exists() { + cfg.compiler(guess); + } + } + } + + _ => {} + } +} + +/// The target programming language for a native compiler. +enum Language { + /// The compiler is targeting C. + C, + /// The compiler is targeting C++. + CPlusPlus, +} + +impl Language { + /// Obtains the name of a compiler in the GCC collection. + fn gcc(self) -> &'static str { + match self { + Language::C => "gcc", + Language::CPlusPlus => "g++", + } + } + + /// Obtains the name of a compiler in the clang suite. + fn clang(self) -> &'static str { + match self { + Language::C => "clang", + Language::CPlusPlus => "clang++", + } + } +} diff --git a/src/bootstrap/channel.rs b/src/bootstrap/channel.rs new file mode 100644 index 000000000..1932a0017 --- /dev/null +++ b/src/bootstrap/channel.rs @@ -0,0 +1,105 @@ +//! Build configuration for Rust's release channels. +//! +//! Implements the stable/beta/nightly channel distinctions by setting various +//! flags like the `unstable_features`, calculating variables like `release` and +//! `package_vers`, and otherwise indicating to the compiler what it should +//! print out as part of its version information. + +use std::path::Path; +use std::process::Command; + +use crate::util::output; +use crate::Build; + +pub enum GitInfo { + /// This is not a git repository. + Absent, + /// This is a git repository. + /// If the info should be used (`ignore_git` is false), this will be + /// `Some`, otherwise it will be `None`. + Present(Option<Info>), +} + +pub struct Info { + commit_date: String, + sha: String, + short_sha: String, +} + +impl GitInfo { + pub fn new(ignore_git: bool, dir: &Path) -> GitInfo { + // See if this even begins to look like a git dir + if !dir.join(".git").exists() { + return GitInfo::Absent; + } + + // Make sure git commands work + match Command::new("git").arg("rev-parse").current_dir(dir).output() { + Ok(ref out) if out.status.success() => {} + _ => return GitInfo::Absent, + } + + // If we're ignoring the git info, we don't actually need to collect it, just make sure this + // was a git repo in the first place. + if ignore_git { + return GitInfo::Present(None); + } + + // Ok, let's scrape some info + let ver_date = output( + Command::new("git") + .current_dir(dir) + .arg("log") + .arg("-1") + .arg("--date=short") + .arg("--pretty=format:%cd"), + ); + let ver_hash = output(Command::new("git").current_dir(dir).arg("rev-parse").arg("HEAD")); + let short_ver_hash = output( + Command::new("git").current_dir(dir).arg("rev-parse").arg("--short=9").arg("HEAD"), + ); + GitInfo::Present(Some(Info { + commit_date: ver_date.trim().to_string(), + sha: ver_hash.trim().to_string(), + short_sha: short_ver_hash.trim().to_string(), + })) + } + + fn info(&self) -> Option<&Info> { + match self { + GitInfo::Present(info) => info.as_ref(), + GitInfo::Absent => None, + } + } + + pub fn sha(&self) -> Option<&str> { + self.info().map(|s| &s.sha[..]) + } + + pub fn sha_short(&self) -> Option<&str> { + self.info().map(|s| &s.short_sha[..]) + } + + pub fn commit_date(&self) -> Option<&str> { + self.info().map(|s| &s.commit_date[..]) + } + + pub fn version(&self, build: &Build, num: &str) -> String { + let mut version = build.release(num); + if let Some(ref inner) = self.info() { + version.push_str(" ("); + version.push_str(&inner.short_sha); + version.push(' '); + version.push_str(&inner.commit_date); + version.push(')'); + } + version + } + + pub fn is_git(&self) -> bool { + match self { + GitInfo::Absent => false, + GitInfo::Present(_) => true, + } + } +} diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs new file mode 100644 index 000000000..4e1e8ef9d --- /dev/null +++ b/src/bootstrap/check.rs @@ -0,0 +1,498 @@ +//! Implementation of compiling the compiler and standard library, in "check"-based modes. + +use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; +use crate::cache::Interned; +use crate::compile::{add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo}; +use crate::config::TargetSelection; +use crate::tool::{prepare_tool_cargo, SourceType}; +use crate::INTERNER; +use crate::{Compiler, Mode, Subcommand}; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Std { + pub target: TargetSelection, +} + +/// Returns args for the subcommand itself (not for cargo) +fn args(builder: &Builder<'_>) -> Vec<String> { + fn strings<'a>(arr: &'a [&str]) -> impl Iterator<Item = String> + 'a { + arr.iter().copied().map(String::from) + } + + if let Subcommand::Clippy { + fix, + clippy_lint_allow, + clippy_lint_deny, + clippy_lint_warn, + clippy_lint_forbid, + .. + } = &builder.config.cmd + { + // disable the most spammy clippy lints + let ignored_lints = vec![ + "many_single_char_names", // there are a lot in stdarch + "collapsible_if", + "type_complexity", + "missing_safety_doc", // almost 3K warnings + "too_many_arguments", + "needless_lifetimes", // people want to keep the lifetimes + "wrong_self_convention", + ]; + let mut args = vec![]; + if *fix { + #[rustfmt::skip] + args.extend(strings(&[ + "--fix", "-Zunstable-options", + // FIXME: currently, `--fix` gives an error while checking tests for libtest, + // possibly because libtest is not yet built in the sysroot. + // As a workaround, avoid checking tests and benches when passed --fix. + "--lib", "--bins", "--examples", + ])); + } + args.extend(strings(&["--", "--cap-lints", "warn"])); + args.extend(ignored_lints.iter().map(|lint| format!("-Aclippy::{}", lint))); + let mut clippy_lint_levels: Vec<String> = Vec::new(); + clippy_lint_allow.iter().for_each(|v| clippy_lint_levels.push(format!("-A{}", v))); + clippy_lint_deny.iter().for_each(|v| clippy_lint_levels.push(format!("-D{}", v))); + clippy_lint_warn.iter().for_each(|v| clippy_lint_levels.push(format!("-W{}", v))); + clippy_lint_forbid.iter().for_each(|v| clippy_lint_levels.push(format!("-F{}", v))); + args.extend(clippy_lint_levels); + args + } else { + vec![] + } +} + +fn cargo_subcommand(kind: Kind) -> &'static str { + match kind { + Kind::Check => "check", + Kind::Clippy => "clippy", + Kind::Fix => "fix", + _ => unreachable!(), + } +} + +impl Step for Std { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.all_krates("test").path("library") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Std { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + builder.update_submodule(&Path::new("library").join("stdarch")); + + let target = self.target; + let compiler = builder.compiler(builder.top_stage, builder.config.build); + + let mut cargo = builder.cargo( + compiler, + Mode::Std, + SourceType::InTree, + target, + cargo_subcommand(builder.kind), + ); + std_cargo(builder, target, compiler.stage, &mut cargo); + + builder.info(&format!( + "Checking stage{} std artifacts ({} -> {})", + builder.top_stage, &compiler.host, target + )); + run_cargo( + builder, + cargo, + args(builder), + &libstd_stamp(builder, compiler, target), + vec![], + true, + ); + + // We skip populating the sysroot in non-zero stage because that'll lead + // to rlib/rmeta conflicts if std gets built during this session. + if compiler.stage == 0 { + let libdir = builder.sysroot_libdir(compiler, target); + let hostdir = builder.sysroot_libdir(compiler, compiler.host); + add_to_sysroot(&builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target)); + } + + // don't run on std twice with x.py clippy + if builder.kind == Kind::Clippy { + return; + } + + // Then run cargo again, once we've put the rmeta files for the library + // crates into the sysroot. This is needed because e.g., core's tests + // depend on `libtest` -- Cargo presumes it will exist, but it doesn't + // since we initialize with an empty sysroot. + // + // Currently only the "libtest" tree of crates does this. + let mut cargo = builder.cargo( + compiler, + Mode::Std, + SourceType::InTree, + target, + cargo_subcommand(builder.kind), + ); + + // If we're not in stage 0, tests and examples will fail to compile + // from `core` definitions being loaded from two different `libcore` + // .rmeta and .rlib files. + if compiler.stage == 0 { + cargo.arg("--all-targets"); + } + + std_cargo(builder, target, compiler.stage, &mut cargo); + + // Explicitly pass -p for all dependencies krates -- this will force cargo + // to also check the tests/benches/examples for these crates, rather + // than just the leaf crate. + for krate in builder.in_tree_crates("test", Some(target)) { + cargo.arg("-p").arg(krate.name); + } + + builder.info(&format!( + "Checking stage{} std test/bench/example targets ({} -> {})", + builder.top_stage, &compiler.host, target + )); + run_cargo( + builder, + cargo, + args(builder), + &libstd_test_stamp(builder, compiler, target), + vec![], + true, + ); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Rustc { + pub target: TargetSelection, +} + +impl Step for Rustc { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.all_krates("rustc-main").path("compiler") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Rustc { target: run.target }); + } + + /// Builds the compiler. + /// + /// This will build the compiler for a particular stage of the build using + /// the `compiler` targeting the `target` architecture. The artifacts + /// created will also be linked into the sysroot directory. + fn run(self, builder: &Builder<'_>) { + let compiler = builder.compiler(builder.top_stage, builder.config.build); + let target = self.target; + + if compiler.stage != 0 { + // If we're not in stage 0, then we won't have a std from the beta + // compiler around. That means we need to make sure there's one in + // the sysroot for the compiler to find. Otherwise, we're going to + // fail when building crates that need to generate code (e.g., build + // scripts and their dependencies). + builder.ensure(crate::compile::Std::new(compiler, compiler.host)); + builder.ensure(crate::compile::Std::new(compiler, target)); + } else { + builder.ensure(Std { target }); + } + + let mut cargo = builder.cargo( + compiler, + Mode::Rustc, + SourceType::InTree, + target, + cargo_subcommand(builder.kind), + ); + rustc_cargo(builder, &mut cargo, target); + + // For ./x.py clippy, don't run with --all-targets because + // linting tests and benchmarks can produce very noisy results + if builder.kind != Kind::Clippy { + cargo.arg("--all-targets"); + } + + // Explicitly pass -p for all compiler krates -- this will force cargo + // to also check the tests/benches/examples for these crates, rather + // than just the leaf crate. + for krate in builder.in_tree_crates("rustc-main", Some(target)) { + cargo.arg("-p").arg(krate.name); + } + + builder.info(&format!( + "Checking stage{} compiler artifacts ({} -> {})", + builder.top_stage, &compiler.host, target + )); + run_cargo( + builder, + cargo, + args(builder), + &librustc_stamp(builder, compiler, target), + vec![], + true, + ); + + let libdir = builder.sysroot_libdir(compiler, target); + let hostdir = builder.sysroot_libdir(compiler, compiler.host); + add_to_sysroot(&builder, &libdir, &hostdir, &librustc_stamp(builder, compiler, target)); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct CodegenBackend { + pub target: TargetSelection, + pub backend: Interned<String>, +} + +impl Step for CodegenBackend { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.paths(&["compiler/rustc_codegen_cranelift", "compiler/rustc_codegen_gcc"]) + } + + fn make_run(run: RunConfig<'_>) { + for &backend in &[INTERNER.intern_str("cranelift"), INTERNER.intern_str("gcc")] { + run.builder.ensure(CodegenBackend { target: run.target, backend }); + } + } + + fn run(self, builder: &Builder<'_>) { + let compiler = builder.compiler(builder.top_stage, builder.config.build); + let target = self.target; + let backend = self.backend; + + builder.ensure(Rustc { target }); + + let mut cargo = builder.cargo( + compiler, + Mode::Codegen, + SourceType::InTree, + target, + cargo_subcommand(builder.kind), + ); + cargo + .arg("--manifest-path") + .arg(builder.src.join(format!("compiler/rustc_codegen_{}/Cargo.toml", backend))); + rustc_cargo_env(builder, &mut cargo, target); + + builder.info(&format!( + "Checking stage{} {} artifacts ({} -> {})", + builder.top_stage, backend, &compiler.host.triple, target.triple + )); + + run_cargo( + builder, + cargo, + args(builder), + &codegen_backend_stamp(builder, compiler, target, backend), + vec![], + true, + ); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct RustAnalyzer { + pub target: TargetSelection, +} + +impl Step for RustAnalyzer { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rust-analyzer") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustAnalyzer { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let compiler = builder.compiler(builder.top_stage, builder.config.build); + let target = self.target; + + builder.ensure(Std { target }); + + let mut cargo = prepare_tool_cargo( + builder, + compiler, + Mode::ToolStd, + target, + cargo_subcommand(builder.kind), + "src/tools/rust-analyzer", + SourceType::InTree, + &["rust-analyzer/in-rust-tree".to_owned()], + ); + + cargo.rustflag( + "-Zallow-features=proc_macro_internals,proc_macro_diagnostic,proc_macro_span", + ); + + // For ./x.py clippy, don't check those targets because + // linting tests and benchmarks can produce very noisy results + if builder.kind != Kind::Clippy { + // can't use `--all-targets` because `--examples` doesn't work well + cargo.arg("--bins"); + cargo.arg("--tests"); + cargo.arg("--benches"); + } + + builder.info(&format!( + "Checking stage{} {} artifacts ({} -> {})", + compiler.stage, "rust-analyzer", &compiler.host.triple, target.triple + )); + run_cargo(builder, cargo, args(builder), &stamp(builder, compiler, target), vec![], true); + + /// Cargo's output path in a given stage, compiled by a particular + /// compiler for the specified target. + fn stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { + builder.cargo_out(compiler, Mode::ToolStd, target).join(".rust-analyzer-check.stamp") + } + } +} + +macro_rules! tool_check_step { + ($name:ident, $path:literal, $($alias:literal, )* $source_type:path $(, $default:literal )?) => { + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] + pub struct $name { + pub target: TargetSelection, + } + + impl Step for $name { + type Output = (); + const ONLY_HOSTS: bool = true; + // don't ever check out-of-tree tools by default, they'll fail when toolstate is broken + const DEFAULT: bool = matches!($source_type, SourceType::InTree) $( && $default )?; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.paths(&[ $path, $($alias),* ]) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure($name { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let compiler = builder.compiler(builder.top_stage, builder.config.build); + let target = self.target; + + builder.ensure(Rustc { target }); + + let mut cargo = prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + target, + cargo_subcommand(builder.kind), + $path, + $source_type, + &[], + ); + + // For ./x.py clippy, don't run with --all-targets because + // linting tests and benchmarks can produce very noisy results + if builder.kind != Kind::Clippy { + cargo.arg("--all-targets"); + } + + // Enable internal lints for clippy and rustdoc + // NOTE: this doesn't enable lints for any other tools unless they explicitly add `#![warn(rustc::internal)]` + // See https://github.com/rust-lang/rust/pull/80573#issuecomment-754010776 + cargo.rustflag("-Zunstable-options"); + + builder.info(&format!( + "Checking stage{} {} artifacts ({} -> {})", + builder.top_stage, + stringify!($name).to_lowercase(), + &compiler.host.triple, + target.triple + )); + run_cargo( + builder, + cargo, + args(builder), + &stamp(builder, compiler, target), + vec![], + true, + ); + + /// Cargo's output path in a given stage, compiled by a particular + /// compiler for the specified target. + fn stamp( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, + ) -> PathBuf { + builder + .cargo_out(compiler, Mode::ToolRustc, target) + .join(format!(".{}-check.stamp", stringify!($name).to_lowercase())) + } + } + } + }; +} + +tool_check_step!(Rustdoc, "src/tools/rustdoc", "src/librustdoc", SourceType::InTree); +// Clippy and Rustfmt are hybrids. They are external tools, but use a git subtree instead +// of a submodule. Since the SourceType only drives the deny-warnings +// behavior, treat it as in-tree so that any new warnings in clippy will be +// rejected. +tool_check_step!(Clippy, "src/tools/clippy", SourceType::InTree); +tool_check_step!(Miri, "src/tools/miri", SourceType::Submodule); +tool_check_step!(Rls, "src/tools/rls", SourceType::Submodule); +tool_check_step!(Rustfmt, "src/tools/rustfmt", SourceType::InTree); + +tool_check_step!(Bootstrap, "src/bootstrap", SourceType::InTree, false); + +/// Cargo's output path for the standard library in a given stage, compiled +/// by a particular compiler for the specified target. +fn libstd_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { + builder.cargo_out(compiler, Mode::Std, target).join(".libstd-check.stamp") +} + +/// Cargo's output path for the standard library in a given stage, compiled +/// by a particular compiler for the specified target. +fn libstd_test_stamp( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, +) -> PathBuf { + builder.cargo_out(compiler, Mode::Std, target).join(".libstd-check-test.stamp") +} + +/// Cargo's output path for librustc in a given stage, compiled by a particular +/// compiler for the specified target. +fn librustc_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { + builder.cargo_out(compiler, Mode::Rustc, target).join(".librustc-check.stamp") +} + +/// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular +/// compiler for the specified target and backend. +fn codegen_backend_stamp( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, + backend: Interned<String>, +) -> PathBuf { + builder + .cargo_out(compiler, Mode::Codegen, target) + .join(format!(".librustc_codegen_{}-check.stamp", backend)) +} diff --git a/src/bootstrap/clean.rs b/src/bootstrap/clean.rs new file mode 100644 index 000000000..069f3d6ac --- /dev/null +++ b/src/bootstrap/clean.rs @@ -0,0 +1,118 @@ +//! Implementation of `make clean` in rustbuild. +//! +//! Responsible for cleaning out a build directory of all old and stale +//! artifacts to prepare for a fresh build. Currently doesn't remove the +//! `build/cache` directory (download cache) or the `build/$target/llvm` +//! directory unless the `--all` flag is present. + +use std::fs; +use std::io::{self, ErrorKind}; +use std::path::Path; + +use crate::util::t; +use crate::Build; + +pub fn clean(build: &Build, all: bool) { + rm_rf("tmp".as_ref()); + + if all { + rm_rf(&build.out); + } else { + rm_rf(&build.out.join("tmp")); + rm_rf(&build.out.join("dist")); + rm_rf(&build.out.join("bootstrap")); + + for host in &build.hosts { + let entries = match build.out.join(host.triple).read_dir() { + Ok(iter) => iter, + Err(_) => continue, + }; + + for entry in entries { + let entry = t!(entry); + if entry.file_name().to_str() == Some("llvm") { + continue; + } + let path = t!(entry.path().canonicalize()); + rm_rf(&path); + } + } + } +} + +fn rm_rf(path: &Path) { + match path.symlink_metadata() { + Err(e) => { + if e.kind() == ErrorKind::NotFound { + return; + } + panic!("failed to get metadata for file {}: {}", path.display(), e); + } + Ok(metadata) => { + if metadata.file_type().is_file() || metadata.file_type().is_symlink() { + do_op(path, "remove file", |p| { + fs::remove_file(p).or_else(|e| { + // Work around the fact that we cannot + // delete an executable while it runs on Windows. + #[cfg(windows)] + if e.kind() == std::io::ErrorKind::PermissionDenied + && p.file_name().and_then(std::ffi::OsStr::to_str) + == Some("bootstrap.exe") + { + eprintln!("warning: failed to delete '{}'.", p.display()); + return Ok(()); + } + Err(e) + }) + }); + return; + } + + for file in t!(fs::read_dir(path)) { + rm_rf(&t!(file).path()); + } + do_op(path, "remove dir", |p| { + fs::remove_dir(p).or_else(|e| { + // Check for dir not empty on Windows + // FIXME: Once `ErrorKind::DirectoryNotEmpty` is stabilized, + // match on `e.kind()` instead. + #[cfg(windows)] + if e.raw_os_error() == Some(145) { + return Ok(()); + } + + Err(e) + }) + }); + } + }; +} + +fn do_op<F>(path: &Path, desc: &str, mut f: F) +where + F: FnMut(&Path) -> io::Result<()>, +{ + match f(path) { + Ok(()) => {} + // On windows we can't remove a readonly file, and git will often clone files as readonly. + // As a result, we have some special logic to remove readonly files on windows. + // This is also the reason that we can't use things like fs::remove_dir_all(). + Err(ref e) if cfg!(windows) && e.kind() == ErrorKind::PermissionDenied => { + let m = t!(path.symlink_metadata()); + let mut p = m.permissions(); + p.set_readonly(false); + t!(fs::set_permissions(path, p)); + f(path).unwrap_or_else(|e| { + // Delete symlinked directories on Windows + #[cfg(windows)] + if m.file_type().is_symlink() && path.is_dir() && fs::remove_dir(path).is_ok() { + return; + } + panic!("failed to {} {}: {}", desc, path.display(), e); + }); + } + Err(e) => { + panic!("failed to {} {}: {}", desc, path.display(), e); + } + } +} diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs new file mode 100644 index 000000000..dd2b9d593 --- /dev/null +++ b/src/bootstrap/compile.rs @@ -0,0 +1,1571 @@ +//! Implementation of compiling various phases of the compiler and standard +//! library. +//! +//! This module contains some of the real meat in the rustbuild build system +//! which is where Cargo is used to compile the standard library, libtest, and +//! the compiler. This module is also responsible for assembling the sysroot as it +//! goes along from the output of the previous stage. + +use std::borrow::Cow; +use std::collections::HashSet; +use std::env; +use std::fs; +use std::io::prelude::*; +use std::io::BufReader; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::str; + +use serde::Deserialize; + +use crate::builder::Cargo; +use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; +use crate::cache::{Interned, INTERNER}; +use crate::config::{LlvmLibunwind, TargetSelection}; +use crate::dist; +use crate::native; +use crate::tool::SourceType; +use crate::util::get_clang_cl_resource_dir; +use crate::util::{exe, is_debug_info, is_dylib, output, symlink_dir, t, up_to_date}; +use crate::LLVM_TOOLS; +use crate::{CLang, Compiler, DependencyType, GitRepo, Mode}; + +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Std { + pub target: TargetSelection, + pub compiler: Compiler, + /// Whether to build only a subset of crates in the standard library. + /// + /// This shouldn't be used from other steps; see the comment on [`Rustc`]. + crates: Interned<Vec<String>>, +} + +impl Std { + pub fn new(compiler: Compiler, target: TargetSelection) -> Self { + Self { target, compiler, crates: Default::default() } + } +} + +/// Return a `-p=x -p=y` string suitable for passing to a cargo invocation. +fn build_crates_in_set(run: &RunConfig<'_>) -> Interned<Vec<String>> { + let mut crates = Vec::new(); + for krate in &run.paths { + let path = krate.assert_single_path(); + let crate_name = run.builder.crate_paths[&path.path]; + crates.push(format!("-p={crate_name}")); + } + INTERNER.intern_list(crates) +} + +impl Step for Std { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + // When downloading stage1, the standard library has already been copied to the sysroot, so + // there's no need to rebuild it. + let builder = run.builder; + run.crate_or_deps("test") + .path("library") + .lazy_default_condition(Box::new(|| !builder.download_rustc())) + } + + fn make_run(run: RunConfig<'_>) { + // Normally, people will pass *just* library if they pass it. + // But it's possible (although strange) to pass something like `library std core`. + // Build all crates anyway, as if they hadn't passed the other args. + let has_library = + run.paths.iter().any(|set| set.assert_single_path().path.ends_with("library")); + let crates = if has_library { Default::default() } else { build_crates_in_set(&run) }; + run.builder.ensure(Std { + compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), + target: run.target, + crates, + }); + } + + /// Builds the standard library. + /// + /// This will build the standard library for a particular stage of the build + /// using the `compiler` targeting the `target` architecture. The artifacts + /// created will also be linked into the sysroot directory. + fn run(self, builder: &Builder<'_>) { + let target = self.target; + let compiler = self.compiler; + + // These artifacts were already copied (in `impl Step for Sysroot`). + // Don't recompile them. + // NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler, + // so its artifacts can't be reused. + if builder.download_rustc() && compiler.stage != 0 { + return; + } + + if builder.config.keep_stage.contains(&compiler.stage) + || builder.config.keep_stage_std.contains(&compiler.stage) + { + builder.info("Warning: Using a potentially old libstd. This may not behave well."); + builder.ensure(StdLink::from_std(self, compiler)); + return; + } + + builder.update_submodule(&Path::new("library").join("stdarch")); + + // Profiler information requires LLVM's compiler-rt + if builder.config.profiler { + builder.update_submodule(&Path::new("src/llvm-project")); + } + + let mut target_deps = builder.ensure(StartupObjects { compiler, target }); + + let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); + if compiler_to_use != compiler { + builder.ensure(Std::new(compiler_to_use, target)); + builder.info(&format!("Uplifting stage1 std ({} -> {})", compiler_to_use.host, target)); + + // Even if we're not building std this stage, the new sysroot must + // still contain the third party objects needed by various targets. + copy_third_party_objects(builder, &compiler, target); + copy_self_contained_objects(builder, &compiler, target); + + builder.ensure(StdLink::from_std(self, compiler_to_use)); + return; + } + + target_deps.extend(copy_third_party_objects(builder, &compiler, target)); + target_deps.extend(copy_self_contained_objects(builder, &compiler, target)); + + let mut cargo = builder.cargo(compiler, Mode::Std, SourceType::InTree, target, "build"); + std_cargo(builder, target, compiler.stage, &mut cargo); + + builder.info(&format!( + "Building stage{} std artifacts ({} -> {})", + compiler.stage, &compiler.host, target + )); + run_cargo( + builder, + cargo, + self.crates.to_vec(), + &libstd_stamp(builder, compiler, target), + target_deps, + false, + ); + + builder.ensure(StdLink::from_std( + self, + builder.compiler(compiler.stage, builder.config.build), + )); + } +} + +fn copy_and_stamp( + builder: &Builder<'_>, + libdir: &Path, + sourcedir: &Path, + name: &str, + target_deps: &mut Vec<(PathBuf, DependencyType)>, + dependency_type: DependencyType, +) { + let target = libdir.join(name); + builder.copy(&sourcedir.join(name), &target); + + target_deps.push((target, dependency_type)); +} + +fn copy_llvm_libunwind(builder: &Builder<'_>, target: TargetSelection, libdir: &Path) -> PathBuf { + let libunwind_path = builder.ensure(native::Libunwind { target }); + let libunwind_source = libunwind_path.join("libunwind.a"); + let libunwind_target = libdir.join("libunwind.a"); + builder.copy(&libunwind_source, &libunwind_target); + libunwind_target +} + +/// Copies third party objects needed by various targets. +fn copy_third_party_objects( + builder: &Builder<'_>, + compiler: &Compiler, + target: TargetSelection, +) -> Vec<(PathBuf, DependencyType)> { + let mut target_deps = vec![]; + + // FIXME: remove this in 2021 + if target == "x86_64-fortanix-unknown-sgx" { + if env::var_os("X86_FORTANIX_SGX_LIBS").is_some() { + builder.info("Warning: X86_FORTANIX_SGX_LIBS environment variable is ignored, libunwind is now compiled as part of rustbuild"); + } + } + + if builder.config.sanitizers_enabled(target) && compiler.stage != 0 { + // The sanitizers are only copied in stage1 or above, + // to avoid creating dependency on LLVM. + target_deps.extend( + copy_sanitizers(builder, &compiler, target) + .into_iter() + .map(|d| (d, DependencyType::Target)), + ); + } + + if target == "x86_64-fortanix-unknown-sgx" + || target.contains("pc-windows-gnullvm") + || builder.config.llvm_libunwind(target) == LlvmLibunwind::InTree + && (target.contains("linux") || target.contains("fuchsia")) + { + let libunwind_path = + copy_llvm_libunwind(builder, target, &builder.sysroot_libdir(*compiler, target)); + target_deps.push((libunwind_path, DependencyType::Target)); + } + + target_deps +} + +/// Copies third party objects needed by various targets for self-contained linkage. +fn copy_self_contained_objects( + builder: &Builder<'_>, + compiler: &Compiler, + target: TargetSelection, +) -> Vec<(PathBuf, DependencyType)> { + let libdir_self_contained = builder.sysroot_libdir(*compiler, target).join("self-contained"); + t!(fs::create_dir_all(&libdir_self_contained)); + let mut target_deps = vec![]; + + // Copies the libc and CRT objects. + // + // rustc historically provides a more self-contained installation for musl targets + // not requiring the presence of a native musl toolchain. For example, it can fall back + // to using gcc from a glibc-targeting toolchain for linking. + // To do that we have to distribute musl startup objects as a part of Rust toolchain + // and link with them manually in the self-contained mode. + if target.contains("musl") { + let srcdir = builder.musl_libdir(target).unwrap_or_else(|| { + panic!("Target {:?} does not have a \"musl-libdir\" key", target.triple) + }); + for &obj in &["libc.a", "crt1.o", "Scrt1.o", "rcrt1.o", "crti.o", "crtn.o"] { + copy_and_stamp( + builder, + &libdir_self_contained, + &srcdir, + obj, + &mut target_deps, + DependencyType::TargetSelfContained, + ); + } + let crt_path = builder.ensure(native::CrtBeginEnd { target }); + for &obj in &["crtbegin.o", "crtbeginS.o", "crtend.o", "crtendS.o"] { + let src = crt_path.join(obj); + let target = libdir_self_contained.join(obj); + builder.copy(&src, &target); + target_deps.push((target, DependencyType::TargetSelfContained)); + } + + if !target.starts_with("s390x") { + let libunwind_path = copy_llvm_libunwind(builder, target, &libdir_self_contained); + target_deps.push((libunwind_path, DependencyType::TargetSelfContained)); + } + } else if target.ends_with("-wasi") { + let srcdir = builder + .wasi_root(target) + .unwrap_or_else(|| { + panic!("Target {:?} does not have a \"wasi-root\" key", target.triple) + }) + .join("lib/wasm32-wasi"); + for &obj in &["libc.a", "crt1-command.o", "crt1-reactor.o"] { + copy_and_stamp( + builder, + &libdir_self_contained, + &srcdir, + obj, + &mut target_deps, + DependencyType::TargetSelfContained, + ); + } + } else if target.ends_with("windows-gnu") { + for obj in ["crt2.o", "dllcrt2.o"].iter() { + let src = compiler_file(builder, builder.cc(target), target, CLang::C, obj); + let target = libdir_self_contained.join(obj); + builder.copy(&src, &target); + target_deps.push((target, DependencyType::TargetSelfContained)); + } + } + + target_deps +} + +/// Configure cargo to compile the standard library, adding appropriate env vars +/// and such. +pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, cargo: &mut Cargo) { + if let Some(target) = env::var_os("MACOSX_STD_DEPLOYMENT_TARGET") { + cargo.env("MACOSX_DEPLOYMENT_TARGET", target); + } + + // Determine if we're going to compile in optimized C intrinsics to + // the `compiler-builtins` crate. These intrinsics live in LLVM's + // `compiler-rt` repository, but our `src/llvm-project` submodule isn't + // always checked out, so we need to conditionally look for this. (e.g. if + // an external LLVM is used we skip the LLVM submodule checkout). + // + // Note that this shouldn't affect the correctness of `compiler-builtins`, + // but only its speed. Some intrinsics in C haven't been translated to Rust + // yet but that's pretty rare. Other intrinsics have optimized + // implementations in C which have only had slower versions ported to Rust, + // so we favor the C version where we can, but it's not critical. + // + // If `compiler-rt` is available ensure that the `c` feature of the + // `compiler-builtins` crate is enabled and it's configured to learn where + // `compiler-rt` is located. + let compiler_builtins_root = builder.src.join("src/llvm-project/compiler-rt"); + let compiler_builtins_c_feature = if compiler_builtins_root.exists() { + // Note that `libprofiler_builtins/build.rs` also computes this so if + // you're changing something here please also change that. + cargo.env("RUST_COMPILER_RT_ROOT", &compiler_builtins_root); + " compiler-builtins-c" + } else { + "" + }; + + if builder.no_std(target) == Some(true) { + let mut features = "compiler-builtins-mem".to_string(); + if !target.starts_with("bpf") { + features.push_str(compiler_builtins_c_feature); + } + + // for no-std targets we only compile a few no_std crates + cargo + .args(&["-p", "alloc"]) + .arg("--manifest-path") + .arg(builder.src.join("library/alloc/Cargo.toml")) + .arg("--features") + .arg(features); + } else { + let mut features = builder.std_features(target); + features.push_str(compiler_builtins_c_feature); + + cargo + .arg("--features") + .arg(features) + .arg("--manifest-path") + .arg(builder.src.join("library/test/Cargo.toml")); + + // Help the libc crate compile by assisting it in finding various + // sysroot native libraries. + if target.contains("musl") { + if let Some(p) = builder.musl_libdir(target) { + let root = format!("native={}", p.to_str().unwrap()); + cargo.rustflag("-L").rustflag(&root); + } + } + + if target.ends_with("-wasi") { + if let Some(p) = builder.wasi_root(target) { + let root = format!("native={}/lib/wasm32-wasi", p.to_str().unwrap()); + cargo.rustflag("-L").rustflag(&root); + } + } + } + + // By default, rustc uses `-Cembed-bitcode=yes`, and Cargo overrides that + // with `-Cembed-bitcode=no` for non-LTO builds. However, libstd must be + // built with bitcode so that the produced rlibs can be used for both LTO + // builds (which use bitcode) and non-LTO builds (which use object code). + // So we override the override here! + // + // But we don't bother for the stage 0 compiler because it's never used + // with LTO. + if stage >= 1 { + cargo.rustflag("-Cembed-bitcode=yes"); + } + + // By default, rustc does not include unwind tables unless they are required + // for a particular target. They are not required by RISC-V targets, but + // compiling the standard library with them means that users can get + // backtraces without having to recompile the standard library themselves. + // + // This choice was discussed in https://github.com/rust-lang/rust/pull/69890 + if target.contains("riscv") { + cargo.rustflag("-Cforce-unwind-tables=yes"); + } + + let html_root = + format!("-Zcrate-attr=doc(html_root_url=\"{}/\")", builder.doc_rust_lang_org_channel(),); + cargo.rustflag(&html_root); + cargo.rustdocflag(&html_root); +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +struct StdLink { + pub compiler: Compiler, + pub target_compiler: Compiler, + pub target: TargetSelection, + /// Not actually used; only present to make sure the cache invalidation is correct. + crates: Interned<Vec<String>>, +} + +impl StdLink { + fn from_std(std: Std, host_compiler: Compiler) -> Self { + Self { + compiler: host_compiler, + target_compiler: std.compiler, + target: std.target, + crates: std.crates, + } + } +} + +impl Step for StdLink { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Link all libstd rlibs/dylibs into the sysroot location. + /// + /// Links those artifacts generated by `compiler` to the `stage` compiler's + /// sysroot for the specified `host` and `target`. + /// + /// Note that this assumes that `compiler` has already generated the libstd + /// libraries for `target`, and this method will find them in the relevant + /// output directory. + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target_compiler = self.target_compiler; + let target = self.target; + builder.info(&format!( + "Copying stage{} std from stage{} ({} -> {} / {})", + target_compiler.stage, compiler.stage, &compiler.host, target_compiler.host, target + )); + let libdir = builder.sysroot_libdir(target_compiler, target); + let hostdir = builder.sysroot_libdir(target_compiler, compiler.host); + add_to_sysroot(builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target)); + } +} + +/// Copies sanitizer runtime libraries into target libdir. +fn copy_sanitizers( + builder: &Builder<'_>, + compiler: &Compiler, + target: TargetSelection, +) -> Vec<PathBuf> { + let runtimes: Vec<native::SanitizerRuntime> = builder.ensure(native::Sanitizers { target }); + + if builder.config.dry_run { + return Vec::new(); + } + + let mut target_deps = Vec::new(); + let libdir = builder.sysroot_libdir(*compiler, target); + + for runtime in &runtimes { + let dst = libdir.join(&runtime.name); + builder.copy(&runtime.path, &dst); + + if target == "x86_64-apple-darwin" || target == "aarch64-apple-darwin" { + // Update the library’s install name to reflect that it has has been renamed. + apple_darwin_update_library_name(&dst, &format!("@rpath/{}", &runtime.name)); + // Upon renaming the install name, the code signature of the file will invalidate, + // so we will sign it again. + apple_darwin_sign_file(&dst); + } + + target_deps.push(dst); + } + + target_deps +} + +fn apple_darwin_update_library_name(library_path: &Path, new_name: &str) { + let status = Command::new("install_name_tool") + .arg("-id") + .arg(new_name) + .arg(library_path) + .status() + .expect("failed to execute `install_name_tool`"); + assert!(status.success()); +} + +fn apple_darwin_sign_file(file_path: &Path) { + let status = Command::new("codesign") + .arg("-f") // Force to rewrite the existing signature + .arg("-s") + .arg("-") + .arg(file_path) + .status() + .expect("failed to execute `codesign`"); + assert!(status.success()); +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct StartupObjects { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for StartupObjects { + type Output = Vec<(PathBuf, DependencyType)>; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("library/rtstartup") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(StartupObjects { + compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), + target: run.target, + }); + } + + /// Builds and prepare startup objects like rsbegin.o and rsend.o + /// + /// These are primarily used on Windows right now for linking executables/dlls. + /// They don't require any library support as they're just plain old object + /// files, so we just use the nightly snapshot compiler to always build them (as + /// no other compilers are guaranteed to be available). + fn run(self, builder: &Builder<'_>) -> Vec<(PathBuf, DependencyType)> { + let for_compiler = self.compiler; + let target = self.target; + if !target.ends_with("windows-gnu") { + return vec![]; + } + + let mut target_deps = vec![]; + + let src_dir = &builder.src.join("library").join("rtstartup"); + let dst_dir = &builder.native_dir(target).join("rtstartup"); + let sysroot_dir = &builder.sysroot_libdir(for_compiler, target); + t!(fs::create_dir_all(dst_dir)); + + for file in &["rsbegin", "rsend"] { + let src_file = &src_dir.join(file.to_string() + ".rs"); + let dst_file = &dst_dir.join(file.to_string() + ".o"); + if !up_to_date(src_file, dst_file) { + let mut cmd = Command::new(&builder.initial_rustc); + cmd.env("RUSTC_BOOTSTRAP", "1"); + if !builder.local_rebuild { + // a local_rebuild compiler already has stage1 features + cmd.arg("--cfg").arg("bootstrap"); + } + builder.run( + cmd.arg("--target") + .arg(target.rustc_target_arg()) + .arg("--emit=obj") + .arg("-o") + .arg(dst_file) + .arg(src_file), + ); + } + + let target = sysroot_dir.join((*file).to_string() + ".o"); + builder.copy(dst_file, &target); + target_deps.push((target, DependencyType::Target)); + } + + target_deps + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Rustc { + pub target: TargetSelection, + pub compiler: Compiler, + /// Whether to build a subset of crates, rather than the whole compiler. + /// + /// This should only be requested by the user, not used within rustbuild itself. + /// Using it within rustbuild can lead to confusing situation where lints are replayed + /// in two different steps. + crates: Interned<Vec<String>>, +} + +impl Rustc { + pub fn new(compiler: Compiler, target: TargetSelection) -> Self { + Self { target, compiler, crates: Default::default() } + } +} + +impl Step for Rustc { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let mut crates = run.builder.in_tree_crates("rustc-main", None); + for (i, krate) in crates.iter().enumerate() { + if krate.name == "rustc-main" { + crates.swap_remove(i); + break; + } + } + run.crates(crates) + } + + fn make_run(run: RunConfig<'_>) { + let crates = build_crates_in_set(&run); + run.builder.ensure(Rustc { + compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), + target: run.target, + crates, + }); + } + + /// Builds the compiler. + /// + /// This will build the compiler for a particular stage of the build using + /// the `compiler` targeting the `target` architecture. The artifacts + /// created will also be linked into the sysroot directory. + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target = self.target; + + // NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler, + // so its artifacts can't be reused. + if builder.download_rustc() && compiler.stage != 0 { + // Copy the existing artifacts instead of rebuilding them. + // NOTE: this path is only taken for tools linking to rustc-dev. + builder.ensure(Sysroot { compiler }); + return; + } + + builder.ensure(Std::new(compiler, target)); + + if builder.config.keep_stage.contains(&compiler.stage) { + builder.info("Warning: Using a potentially old librustc. This may not behave well."); + builder.info("Warning: Use `--keep-stage-std` if you want to rebuild the compiler when it changes"); + builder.ensure(RustcLink::from_rustc(self, compiler)); + return; + } + + let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); + if compiler_to_use != compiler { + builder.ensure(Rustc::new(compiler_to_use, target)); + builder + .info(&format!("Uplifting stage1 rustc ({} -> {})", builder.config.build, target)); + builder.ensure(RustcLink::from_rustc(self, compiler_to_use)); + return; + } + + // Ensure that build scripts and proc macros have a std / libproc_macro to link against. + builder.ensure(Std::new( + builder.compiler(self.compiler.stage, builder.config.build), + builder.config.build, + )); + + let mut cargo = builder.cargo(compiler, Mode::Rustc, SourceType::InTree, target, "build"); + rustc_cargo(builder, &mut cargo, target); + + if builder.config.rust_profile_use.is_some() + && builder.config.rust_profile_generate.is_some() + { + panic!("Cannot use and generate PGO profiles at the same time"); + } + + // With LLD, we can use ICF (identical code folding) to reduce the executable size + // of librustc_driver/rustc and to improve i-cache utilization. + if builder.config.use_lld { + cargo.rustflag("-Clink-args=-Wl,--icf=all"); + } + + let is_collecting = if let Some(path) = &builder.config.rust_profile_generate { + if compiler.stage == 1 { + cargo.rustflag(&format!("-Cprofile-generate={}", path)); + // Apparently necessary to avoid overflowing the counters during + // a Cargo build profile + cargo.rustflag("-Cllvm-args=-vp-counters-per-site=4"); + true + } else { + false + } + } else if let Some(path) = &builder.config.rust_profile_use { + if compiler.stage == 1 { + cargo.rustflag(&format!("-Cprofile-use={}", path)); + cargo.rustflag("-Cllvm-args=-pgo-warn-missing-function"); + true + } else { + false + } + } else { + false + }; + if is_collecting { + // Ensure paths to Rust sources are relative, not absolute. + cargo.rustflag(&format!( + "-Cllvm-args=-static-func-strip-dirname-prefix={}", + builder.config.src.components().count() + )); + } + + builder.info(&format!( + "Building stage{} compiler artifacts ({} -> {})", + compiler.stage, &compiler.host, target + )); + run_cargo( + builder, + cargo, + self.crates.to_vec(), + &librustc_stamp(builder, compiler, target), + vec![], + false, + ); + + builder.ensure(RustcLink::from_rustc( + self, + builder.compiler(compiler.stage, builder.config.build), + )); + } +} + +pub fn rustc_cargo(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelection) { + cargo + .arg("--features") + .arg(builder.rustc_features(builder.kind)) + .arg("--manifest-path") + .arg(builder.src.join("compiler/rustc/Cargo.toml")); + rustc_cargo_env(builder, cargo, target); +} + +pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelection) { + // Set some configuration variables picked up by build scripts and + // the compiler alike + cargo + .env("CFG_RELEASE", builder.rust_release()) + .env("CFG_RELEASE_CHANNEL", &builder.config.channel) + .env("CFG_VERSION", builder.rust_version()); + + if let Some(backend) = builder.config.rust_codegen_backends.get(0) { + cargo.env("CFG_DEFAULT_CODEGEN_BACKEND", backend); + } + + let libdir_relative = builder.config.libdir_relative().unwrap_or_else(|| Path::new("lib")); + let target_config = builder.config.target_config.get(&target); + + cargo.env("CFG_LIBDIR_RELATIVE", libdir_relative); + + if let Some(ref ver_date) = builder.rust_info.commit_date() { + cargo.env("CFG_VER_DATE", ver_date); + } + if let Some(ref ver_hash) = builder.rust_info.sha() { + cargo.env("CFG_VER_HASH", ver_hash); + } + if !builder.unstable_features() { + cargo.env("CFG_DISABLE_UNSTABLE_FEATURES", "1"); + } + + // Prefer the current target's own default_linker, else a globally + // specified one. + if let Some(s) = target_config.and_then(|c| c.default_linker.as_ref()) { + cargo.env("CFG_DEFAULT_LINKER", s); + } else if let Some(ref s) = builder.config.rustc_default_linker { + cargo.env("CFG_DEFAULT_LINKER", s); + } + + if builder.config.rustc_parallel { + // keep in sync with `bootstrap/lib.rs:Build::rustc_features` + // `cfg` option for rustc, `features` option for cargo, for conditional compilation + cargo.rustflag("--cfg=parallel_compiler"); + cargo.rustdocflag("--cfg=parallel_compiler"); + } + if builder.config.rust_verify_llvm_ir { + cargo.env("RUSTC_VERIFY_LLVM_IR", "1"); + } + + // Pass down configuration from the LLVM build into the build of + // rustc_llvm and rustc_codegen_llvm. + // + // Note that this is disabled if LLVM itself is disabled or we're in a check + // build. If we are in a check build we still go ahead here presuming we've + // detected that LLVM is already built and good to go which helps prevent + // busting caches (e.g. like #71152). + if builder.config.llvm_enabled() + && (builder.kind != Kind::Check + || crate::native::prebuilt_llvm_config(builder, target).is_ok()) + { + if builder.is_rust_llvm(target) { + cargo.env("LLVM_RUSTLLVM", "1"); + } + let llvm_config = builder.ensure(native::Llvm { target }); + cargo.env("LLVM_CONFIG", &llvm_config); + if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) { + cargo.env("CFG_LLVM_ROOT", s); + } + + // Some LLVM linker flags (-L and -l) may be needed to link `rustc_llvm`. Its build script + // expects these to be passed via the `LLVM_LINKER_FLAGS` env variable, separated by + // whitespace. + // + // For example: + // - on windows, when `clang-cl` is used with instrumentation, we need to manually add + // clang's runtime library resource directory so that the profiler runtime library can be + // found. This is to avoid the linker errors about undefined references to + // `__llvm_profile_instrument_memop` when linking `rustc_driver`. + let mut llvm_linker_flags = String::new(); + if builder.config.llvm_profile_generate && target.contains("msvc") { + if let Some(ref clang_cl_path) = builder.config.llvm_clang_cl { + // Add clang's runtime library directory to the search path + let clang_rt_dir = get_clang_cl_resource_dir(clang_cl_path); + llvm_linker_flags.push_str(&format!("-L{}", clang_rt_dir.display())); + } + } + + // The config can also specify its own llvm linker flags. + if let Some(ref s) = builder.config.llvm_ldflags { + if !llvm_linker_flags.is_empty() { + llvm_linker_flags.push_str(" "); + } + llvm_linker_flags.push_str(s); + } + + // Set the linker flags via the env var that `rustc_llvm`'s build script will read. + if !llvm_linker_flags.is_empty() { + cargo.env("LLVM_LINKER_FLAGS", llvm_linker_flags); + } + + // Building with a static libstdc++ is only supported on linux right now, + // not for MSVC or macOS + if builder.config.llvm_static_stdcpp + && !target.contains("freebsd") + && !target.contains("msvc") + && !target.contains("apple") + && !target.contains("solaris") + { + let file = compiler_file( + builder, + builder.cxx(target).unwrap(), + target, + CLang::Cxx, + "libstdc++.a", + ); + cargo.env("LLVM_STATIC_STDCPP", file); + } + if builder.llvm_link_shared() { + cargo.env("LLVM_LINK_SHARED", "1"); + } + if builder.config.llvm_use_libcxx { + cargo.env("LLVM_USE_LIBCXX", "1"); + } + if builder.config.llvm_optimize && !builder.config.llvm_release_debuginfo { + cargo.env("LLVM_NDEBUG", "1"); + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +struct RustcLink { + pub compiler: Compiler, + pub target_compiler: Compiler, + pub target: TargetSelection, + /// Not actually used; only present to make sure the cache invalidation is correct. + crates: Interned<Vec<String>>, +} + +impl RustcLink { + fn from_rustc(rustc: Rustc, host_compiler: Compiler) -> Self { + Self { + compiler: host_compiler, + target_compiler: rustc.compiler, + target: rustc.target, + crates: rustc.crates, + } + } +} + +impl Step for RustcLink { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Same as `std_link`, only for librustc + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target_compiler = self.target_compiler; + let target = self.target; + builder.info(&format!( + "Copying stage{} rustc from stage{} ({} -> {} / {})", + target_compiler.stage, compiler.stage, &compiler.host, target_compiler.host, target + )); + add_to_sysroot( + builder, + &builder.sysroot_libdir(target_compiler, target), + &builder.sysroot_libdir(target_compiler, compiler.host), + &librustc_stamp(builder, compiler, target), + ); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct CodegenBackend { + pub target: TargetSelection, + pub compiler: Compiler, + pub backend: Interned<String>, +} + +impl Step for CodegenBackend { + type Output = (); + const ONLY_HOSTS: bool = true; + // Only the backends specified in the `codegen-backends` entry of `config.toml` are built. + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.paths(&["compiler/rustc_codegen_cranelift", "compiler/rustc_codegen_gcc"]) + } + + fn make_run(run: RunConfig<'_>) { + for &backend in &run.builder.config.rust_codegen_backends { + if backend == "llvm" { + continue; // Already built as part of rustc + } + + run.builder.ensure(CodegenBackend { + target: run.target, + compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), + backend, + }); + } + } + + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target = self.target; + let backend = self.backend; + + builder.ensure(Rustc::new(compiler, target)); + + if builder.config.keep_stage.contains(&compiler.stage) { + builder.info( + "Warning: Using a potentially old codegen backend. \ + This may not behave well.", + ); + // Codegen backends are linked separately from this step today, so we don't do + // anything here. + return; + } + + let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); + if compiler_to_use != compiler { + builder.ensure(CodegenBackend { compiler: compiler_to_use, target, backend }); + return; + } + + let out_dir = builder.cargo_out(compiler, Mode::Codegen, target); + + let mut cargo = builder.cargo(compiler, Mode::Codegen, SourceType::InTree, target, "build"); + cargo + .arg("--manifest-path") + .arg(builder.src.join(format!("compiler/rustc_codegen_{}/Cargo.toml", backend))); + rustc_cargo_env(builder, &mut cargo, target); + + let tmp_stamp = out_dir.join(".tmp.stamp"); + + builder.info(&format!( + "Building stage{} codegen backend {} ({} -> {})", + compiler.stage, backend, &compiler.host, target + )); + let files = run_cargo(builder, cargo, vec![], &tmp_stamp, vec![], false); + if builder.config.dry_run { + return; + } + let mut files = files.into_iter().filter(|f| { + let filename = f.file_name().unwrap().to_str().unwrap(); + is_dylib(filename) && filename.contains("rustc_codegen_") + }); + let codegen_backend = match files.next() { + Some(f) => f, + None => panic!("no dylibs built for codegen backend?"), + }; + if let Some(f) = files.next() { + panic!( + "codegen backend built two dylibs:\n{}\n{}", + codegen_backend.display(), + f.display() + ); + } + let stamp = codegen_backend_stamp(builder, compiler, target, backend); + let codegen_backend = codegen_backend.to_str().unwrap(); + t!(fs::write(&stamp, &codegen_backend)); + } +} + +/// Creates the `codegen-backends` folder for a compiler that's about to be +/// assembled as a complete compiler. +/// +/// This will take the codegen artifacts produced by `compiler` and link them +/// into an appropriate location for `target_compiler` to be a functional +/// compiler. +fn copy_codegen_backends_to_sysroot( + builder: &Builder<'_>, + compiler: Compiler, + target_compiler: Compiler, +) { + let target = target_compiler.host; + + // Note that this step is different than all the other `*Link` steps in + // that it's not assembling a bunch of libraries but rather is primarily + // moving the codegen backend into place. The codegen backend of rustc is + // not linked into the main compiler by default but is rather dynamically + // selected at runtime for inclusion. + // + // Here we're looking for the output dylib of the `CodegenBackend` step and + // we're copying that into the `codegen-backends` folder. + let dst = builder.sysroot_codegen_backends(target_compiler); + t!(fs::create_dir_all(&dst), dst); + + if builder.config.dry_run { + return; + } + + for backend in builder.config.rust_codegen_backends.iter() { + if backend == "llvm" { + continue; // Already built as part of rustc + } + + let stamp = codegen_backend_stamp(builder, compiler, target, *backend); + let dylib = t!(fs::read_to_string(&stamp)); + let file = Path::new(&dylib); + let filename = file.file_name().unwrap().to_str().unwrap(); + // change `librustc_codegen_cranelift-xxxxxx.so` to + // `librustc_codegen_cranelift-release.so` + let target_filename = { + let dash = filename.find('-').unwrap(); + let dot = filename.find('.').unwrap(); + format!("{}-{}{}", &filename[..dash], builder.rust_release(), &filename[dot..]) + }; + builder.copy(&file, &dst.join(target_filename)); + } +} + +/// Cargo's output path for the standard library in a given stage, compiled +/// by a particular compiler for the specified target. +pub fn libstd_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { + builder.cargo_out(compiler, Mode::Std, target).join(".libstd.stamp") +} + +/// Cargo's output path for librustc in a given stage, compiled by a particular +/// compiler for the specified target. +pub fn librustc_stamp( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, +) -> PathBuf { + builder.cargo_out(compiler, Mode::Rustc, target).join(".librustc.stamp") +} + +/// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular +/// compiler for the specified target and backend. +fn codegen_backend_stamp( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, + backend: Interned<String>, +) -> PathBuf { + builder + .cargo_out(compiler, Mode::Codegen, target) + .join(format!(".librustc_codegen_{}.stamp", backend)) +} + +pub fn compiler_file( + builder: &Builder<'_>, + compiler: &Path, + target: TargetSelection, + c: CLang, + file: &str, +) -> PathBuf { + let mut cmd = Command::new(compiler); + cmd.args(builder.cflags(target, GitRepo::Rustc, c)); + cmd.arg(format!("-print-file-name={}", file)); + let out = output(&mut cmd); + PathBuf::from(out.trim()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Sysroot { + pub compiler: Compiler, +} + +impl Step for Sysroot { + type Output = Interned<PathBuf>; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Returns the sysroot for the `compiler` specified that *this build system + /// generates*. + /// + /// That is, the sysroot for the stage0 compiler is not what the compiler + /// thinks it is by default, but it's the same as the default for stages + /// 1-3. + fn run(self, builder: &Builder<'_>) -> Interned<PathBuf> { + let compiler = self.compiler; + let sysroot = if compiler.stage == 0 { + builder.out.join(&compiler.host.triple).join("stage0-sysroot") + } else { + builder.out.join(&compiler.host.triple).join(format!("stage{}", compiler.stage)) + }; + let _ = fs::remove_dir_all(&sysroot); + t!(fs::create_dir_all(&sysroot)); + + // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. + if builder.download_rustc() && compiler.stage != 0 { + assert_eq!( + builder.config.build, compiler.host, + "Cross-compiling is not yet supported with `download-rustc`", + ); + // Copy the compiler into the correct sysroot. + let ci_rustc_dir = + builder.config.out.join(&*builder.config.build.triple).join("ci-rustc"); + builder.cp_r(&ci_rustc_dir, &sysroot); + return INTERNER.intern_path(sysroot); + } + + // Symlink the source root into the same location inside the sysroot, + // where `rust-src` component would go (`$sysroot/lib/rustlib/src/rust`), + // so that any tools relying on `rust-src` also work for local builds, + // and also for translating the virtual `/rustc/$hash` back to the real + // directory (for running tests with `rust.remap-debuginfo = true`). + let sysroot_lib_rustlib_src = sysroot.join("lib/rustlib/src"); + t!(fs::create_dir_all(&sysroot_lib_rustlib_src)); + let sysroot_lib_rustlib_src_rust = sysroot_lib_rustlib_src.join("rust"); + if let Err(e) = symlink_dir(&builder.config, &builder.src, &sysroot_lib_rustlib_src_rust) { + eprintln!( + "warning: creating symbolic link `{}` to `{}` failed with {}", + sysroot_lib_rustlib_src_rust.display(), + builder.src.display(), + e, + ); + if builder.config.rust_remap_debuginfo { + eprintln!( + "warning: some `src/test/ui` tests will fail when lacking `{}`", + sysroot_lib_rustlib_src_rust.display(), + ); + } + } + + INTERNER.intern_path(sysroot) + } +} + +#[derive(Debug, Copy, PartialOrd, Ord, Clone, PartialEq, Eq, Hash)] +pub struct Assemble { + /// The compiler which we will produce in this step. Assemble itself will + /// take care of ensuring that the necessary prerequisites to do so exist, + /// that is, this target can be a stage2 compiler and Assemble will build + /// previous stages for you. + pub target_compiler: Compiler, +} + +impl Step for Assemble { + type Output = Compiler; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("compiler/rustc").path("compiler") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Assemble { + target_compiler: run.builder.compiler(run.builder.top_stage + 1, run.target), + }); + } + + /// Prepare a new compiler from the artifacts in `stage` + /// + /// This will assemble a compiler in `build/$host/stage$stage`. The compiler + /// must have been previously produced by the `stage - 1` builder.build + /// compiler. + fn run(self, builder: &Builder<'_>) -> Compiler { + let target_compiler = self.target_compiler; + + if target_compiler.stage == 0 { + assert_eq!( + builder.config.build, target_compiler.host, + "Cannot obtain compiler for non-native build triple at stage 0" + ); + // The stage 0 compiler for the build triple is always pre-built. + return target_compiler; + } + + // Get the compiler that we'll use to bootstrap ourselves. + // + // Note that this is where the recursive nature of the bootstrap + // happens, as this will request the previous stage's compiler on + // downwards to stage 0. + // + // Also note that we're building a compiler for the host platform. We + // only assume that we can run `build` artifacts, which means that to + // produce some other architecture compiler we need to start from + // `build` to get there. + // + // FIXME: It may be faster if we build just a stage 1 compiler and then + // use that to bootstrap this compiler forward. + let build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build); + + // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. + if builder.download_rustc() { + builder.ensure(Sysroot { compiler: target_compiler }); + return target_compiler; + } + + // Build the libraries for this compiler to link to (i.e., the libraries + // it uses at runtime). NOTE: Crates the target compiler compiles don't + // link to these. (FIXME: Is that correct? It seems to be correct most + // of the time but I think we do link to these for stage2/bin compilers + // when not performing a full bootstrap). + builder.ensure(Rustc::new(build_compiler, target_compiler.host)); + + for &backend in builder.config.rust_codegen_backends.iter() { + if backend == "llvm" { + continue; // Already built as part of rustc + } + + builder.ensure(CodegenBackend { + compiler: build_compiler, + target: target_compiler.host, + backend, + }); + } + + let lld_install = if builder.config.lld_enabled { + Some(builder.ensure(native::Lld { target: target_compiler.host })) + } else { + None + }; + + let stage = target_compiler.stage; + let host = target_compiler.host; + builder.info(&format!("Assembling stage{} compiler ({})", stage, host)); + + // Link in all dylibs to the libdir + let stamp = librustc_stamp(builder, build_compiler, target_compiler.host); + let proc_macros = builder + .read_stamp_file(&stamp) + .into_iter() + .filter_map(|(path, dependency_type)| { + if dependency_type == DependencyType::Host { + Some(path.file_name().unwrap().to_owned().into_string().unwrap()) + } else { + None + } + }) + .collect::<HashSet<_>>(); + + let sysroot = builder.sysroot(target_compiler); + let rustc_libdir = builder.rustc_libdir(target_compiler); + t!(fs::create_dir_all(&rustc_libdir)); + let src_libdir = builder.sysroot_libdir(build_compiler, host); + for f in builder.read_dir(&src_libdir) { + let filename = f.file_name().into_string().unwrap(); + if (is_dylib(&filename) || is_debug_info(&filename)) && !proc_macros.contains(&filename) + { + builder.copy(&f.path(), &rustc_libdir.join(&filename)); + } + } + + copy_codegen_backends_to_sysroot(builder, build_compiler, target_compiler); + + // We prepend this bin directory to the user PATH when linking Rust binaries. To + // avoid shadowing the system LLD we rename the LLD we provide to `rust-lld`. + let libdir = builder.sysroot_libdir(target_compiler, target_compiler.host); + let libdir_bin = libdir.parent().unwrap().join("bin"); + t!(fs::create_dir_all(&libdir_bin)); + if let Some(lld_install) = lld_install { + let src_exe = exe("lld", target_compiler.host); + let dst_exe = exe("rust-lld", target_compiler.host); + builder.copy(&lld_install.join("bin").join(&src_exe), &libdir_bin.join(&dst_exe)); + // for `-Z gcc-ld=lld` + let gcc_ld_dir = libdir_bin.join("gcc-ld"); + t!(fs::create_dir(&gcc_ld_dir)); + let lld_wrapper_exe = builder.ensure(crate::tool::LldWrapper { + compiler: build_compiler, + target: target_compiler.host, + }); + builder.copy(&lld_wrapper_exe, &gcc_ld_dir.join(exe("ld", target_compiler.host))); + } + + if builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) { + let llvm_config_bin = builder.ensure(native::Llvm { target: target_compiler.host }); + if !builder.config.dry_run { + let llvm_bin_dir = output(Command::new(llvm_config_bin).arg("--bindir")); + let llvm_bin_dir = Path::new(llvm_bin_dir.trim()); + + // Since we've already built the LLVM tools, install them to the sysroot. + // This is the equivalent of installing the `llvm-tools-preview` component via + // rustup, and lets developers use a locally built toolchain to + // build projects that expect llvm tools to be present in the sysroot + // (e.g. the `bootimage` crate). + for tool in LLVM_TOOLS { + let tool_exe = exe(tool, target_compiler.host); + let src_path = llvm_bin_dir.join(&tool_exe); + // When using `download-ci-llvm`, some of the tools + // may not exist, so skip trying to copy them. + if src_path.exists() { + builder.copy(&src_path, &libdir_bin.join(&tool_exe)); + } + } + } + } + + // Ensure that `libLLVM.so` ends up in the newly build compiler directory, + // so that it can be found when the newly built `rustc` is run. + dist::maybe_install_llvm_runtime(builder, target_compiler.host, &sysroot); + dist::maybe_install_llvm_target(builder, target_compiler.host, &sysroot); + + // Link the compiler binary itself into place + let out_dir = builder.cargo_out(build_compiler, Mode::Rustc, host); + let rustc = out_dir.join(exe("rustc-main", host)); + let bindir = sysroot.join("bin"); + t!(fs::create_dir_all(&bindir)); + let compiler = builder.rustc(target_compiler); + builder.copy(&rustc, &compiler); + + target_compiler + } +} + +/// Link some files into a rustc sysroot. +/// +/// For a particular stage this will link the file listed in `stamp` into the +/// `sysroot_dst` provided. +pub fn add_to_sysroot( + builder: &Builder<'_>, + sysroot_dst: &Path, + sysroot_host_dst: &Path, + stamp: &Path, +) { + let self_contained_dst = &sysroot_dst.join("self-contained"); + t!(fs::create_dir_all(&sysroot_dst)); + t!(fs::create_dir_all(&sysroot_host_dst)); + t!(fs::create_dir_all(&self_contained_dst)); + for (path, dependency_type) in builder.read_stamp_file(stamp) { + let dst = match dependency_type { + DependencyType::Host => sysroot_host_dst, + DependencyType::Target => sysroot_dst, + DependencyType::TargetSelfContained => self_contained_dst, + }; + builder.copy(&path, &dst.join(path.file_name().unwrap())); + } +} + +pub fn run_cargo( + builder: &Builder<'_>, + cargo: Cargo, + tail_args: Vec<String>, + stamp: &Path, + additional_target_deps: Vec<(PathBuf, DependencyType)>, + is_check: bool, +) -> Vec<PathBuf> { + if builder.config.dry_run { + return Vec::new(); + } + + // `target_root_dir` looks like $dir/$target/release + let target_root_dir = stamp.parent().unwrap(); + // `target_deps_dir` looks like $dir/$target/release/deps + let target_deps_dir = target_root_dir.join("deps"); + // `host_root_dir` looks like $dir/release + let host_root_dir = target_root_dir + .parent() + .unwrap() // chop off `release` + .parent() + .unwrap() // chop off `$target` + .join(target_root_dir.file_name().unwrap()); + + // Spawn Cargo slurping up its JSON output. We'll start building up the + // `deps` array of all files it generated along with a `toplevel` array of + // files we need to probe for later. + let mut deps = Vec::new(); + let mut toplevel = Vec::new(); + let ok = stream_cargo(builder, cargo, tail_args, &mut |msg| { + let (filenames, crate_types) = match msg { + CargoMessage::CompilerArtifact { + filenames, + target: CargoTarget { crate_types }, + .. + } => (filenames, crate_types), + _ => return, + }; + for filename in filenames { + // Skip files like executables + if !(filename.ends_with(".rlib") + || filename.ends_with(".lib") + || filename.ends_with(".a") + || is_debug_info(&filename) + || is_dylib(&filename) + || (is_check && filename.ends_with(".rmeta"))) + { + continue; + } + + let filename = Path::new(&*filename); + + // If this was an output file in the "host dir" we don't actually + // worry about it, it's not relevant for us + if filename.starts_with(&host_root_dir) { + // Unless it's a proc macro used in the compiler + if crate_types.iter().any(|t| t == "proc-macro") { + deps.push((filename.to_path_buf(), DependencyType::Host)); + } + continue; + } + + // If this was output in the `deps` dir then this is a precise file + // name (hash included) so we start tracking it. + if filename.starts_with(&target_deps_dir) { + deps.push((filename.to_path_buf(), DependencyType::Target)); + continue; + } + + // Otherwise this was a "top level artifact" which right now doesn't + // have a hash in the name, but there's a version of this file in + // the `deps` folder which *does* have a hash in the name. That's + // the one we'll want to we'll probe for it later. + // + // We do not use `Path::file_stem` or `Path::extension` here, + // because some generated files may have multiple extensions e.g. + // `std-<hash>.dll.lib` on Windows. The aforementioned methods only + // split the file name by the last extension (`.lib`) while we need + // to split by all extensions (`.dll.lib`). + let expected_len = t!(filename.metadata()).len(); + let filename = filename.file_name().unwrap().to_str().unwrap(); + let mut parts = filename.splitn(2, '.'); + let file_stem = parts.next().unwrap().to_owned(); + let extension = parts.next().unwrap().to_owned(); + + toplevel.push((file_stem, extension, expected_len)); + } + }); + + if !ok { + crate::detail_exit(1); + } + + // Ok now we need to actually find all the files listed in `toplevel`. We've + // got a list of prefix/extensions and we basically just need to find the + // most recent file in the `deps` folder corresponding to each one. + let contents = t!(target_deps_dir.read_dir()) + .map(|e| t!(e)) + .map(|e| (e.path(), e.file_name().into_string().unwrap(), t!(e.metadata()))) + .collect::<Vec<_>>(); + for (prefix, extension, expected_len) in toplevel { + let candidates = contents.iter().filter(|&&(_, ref filename, ref meta)| { + meta.len() == expected_len + && filename + .strip_prefix(&prefix[..]) + .map(|s| s.starts_with('-') && s.ends_with(&extension[..])) + .unwrap_or(false) + }); + let max = candidates.max_by_key(|&&(_, _, ref metadata)| { + metadata.modified().expect("mtime should be available on all relevant OSes") + }); + let path_to_add = match max { + Some(triple) => triple.0.to_str().unwrap(), + None => panic!("no output generated for {:?} {:?}", prefix, extension), + }; + if is_dylib(path_to_add) { + let candidate = format!("{}.lib", path_to_add); + let candidate = PathBuf::from(candidate); + if candidate.exists() { + deps.push((candidate, DependencyType::Target)); + } + } + deps.push((path_to_add.into(), DependencyType::Target)); + } + + deps.extend(additional_target_deps); + deps.sort(); + let mut new_contents = Vec::new(); + for (dep, dependency_type) in deps.iter() { + new_contents.extend(match *dependency_type { + DependencyType::Host => b"h", + DependencyType::Target => b"t", + DependencyType::TargetSelfContained => b"s", + }); + new_contents.extend(dep.to_str().unwrap().as_bytes()); + new_contents.extend(b"\0"); + } + t!(fs::write(&stamp, &new_contents)); + deps.into_iter().map(|(d, _)| d).collect() +} + +pub fn stream_cargo( + builder: &Builder<'_>, + cargo: Cargo, + tail_args: Vec<String>, + cb: &mut dyn FnMut(CargoMessage<'_>), +) -> bool { + let mut cargo = Command::from(cargo); + if builder.config.dry_run { + return true; + } + // Instruct Cargo to give us json messages on stdout, critically leaving + // stderr as piped so we can get those pretty colors. + let mut message_format = if builder.config.json_output { + String::from("json") + } else { + String::from("json-render-diagnostics") + }; + if let Some(s) = &builder.config.rustc_error_format { + message_format.push_str(",json-diagnostic-"); + message_format.push_str(s); + } + cargo.arg("--message-format").arg(message_format).stdout(Stdio::piped()); + + for arg in tail_args { + cargo.arg(arg); + } + + builder.verbose(&format!("running: {:?}", cargo)); + let mut child = match cargo.spawn() { + Ok(child) => child, + Err(e) => panic!("failed to execute command: {:?}\nerror: {}", cargo, e), + }; + + // Spawn Cargo slurping up its JSON output. We'll start building up the + // `deps` array of all files it generated along with a `toplevel` array of + // files we need to probe for later. + let stdout = BufReader::new(child.stdout.take().unwrap()); + for line in stdout.lines() { + let line = t!(line); + match serde_json::from_str::<CargoMessage<'_>>(&line) { + Ok(msg) => { + if builder.config.json_output { + // Forward JSON to stdout. + println!("{}", line); + } + cb(msg) + } + // If this was informational, just print it out and continue + Err(_) => println!("{}", line), + } + } + + // Make sure Cargo actually succeeded after we read all of its stdout. + let status = t!(child.wait()); + if builder.is_verbose() && !status.success() { + eprintln!( + "command did not execute successfully: {:?}\n\ + expected success, got: {}", + cargo, status + ); + } + status.success() +} + +#[derive(Deserialize)] +pub struct CargoTarget<'a> { + crate_types: Vec<Cow<'a, str>>, +} + +#[derive(Deserialize)] +#[serde(tag = "reason", rename_all = "kebab-case")] +pub enum CargoMessage<'a> { + CompilerArtifact { + package_id: Cow<'a, str>, + features: Vec<Cow<'a, str>>, + filenames: Vec<Cow<'a, str>>, + target: CargoTarget<'a>, + }, + BuildScriptExecuted { + package_id: Cow<'a, str>, + }, + BuildFinished { + success: bool, + }, +} diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs new file mode 100644 index 000000000..4325a237c --- /dev/null +++ b/src/bootstrap/config.rs @@ -0,0 +1,1668 @@ +//! Serialized configuration of a build. +//! +//! This module implements parsing `config.toml` configuration files to tweak +//! how the build runs. + +use std::cell::{Cell, RefCell}; +use std::cmp; +use std::collections::{HashMap, HashSet}; +use std::env; +use std::ffi::OsStr; +use std::fmt; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::str::FromStr; + +use crate::builder::{Builder, TaskPath}; +use crate::cache::{Interned, INTERNER}; +use crate::channel::GitInfo; +pub use crate::flags::Subcommand; +use crate::flags::{Color, Flags}; +use crate::util::{exe, output, program_out_of_date, t}; +use once_cell::sync::OnceCell; +use serde::{Deserialize, Deserializer}; + +macro_rules! check_ci_llvm { + ($name:expr) => { + assert!( + $name.is_none(), + "setting {} is incompatible with download-ci-llvm.", + stringify!($name) + ); + }; +} + +/// Global configuration for the entire build and/or bootstrap. +/// +/// This structure is derived from a combination of both `config.toml` and +/// `config.mk`. As of the time of this writing it's unlikely that `config.toml` +/// is used all that much, so this is primarily filled out by `config.mk` which +/// is generated from `./configure`. +/// +/// Note that this structure is not decoded directly into, but rather it is +/// filled out from the decoded forms of the structs below. For documentation +/// each field, see the corresponding fields in +/// `config.toml.example`. +#[derive(Default)] +#[cfg_attr(test, derive(Clone))] +pub struct Config { + pub changelog_seen: Option<usize>, + pub ccache: Option<String>, + /// Call Build::ninja() instead of this. + pub ninja_in_file: bool, + pub verbose: usize, + pub submodules: Option<bool>, + pub compiler_docs: bool, + pub docs_minification: bool, + pub docs: bool, + pub locked_deps: bool, + pub vendor: bool, + pub target_config: HashMap<TargetSelection, Target>, + pub full_bootstrap: bool, + pub extended: bool, + pub tools: Option<HashSet<String>>, + pub sanitizers: bool, + pub profiler: bool, + pub ignore_git: bool, + pub exclude: Vec<TaskPath>, + pub include_default_paths: bool, + pub rustc_error_format: Option<String>, + pub json_output: bool, + pub test_compare_mode: bool, + pub color: Color, + pub patch_binaries_for_nix: bool, + pub stage0_metadata: Stage0Metadata, + + pub on_fail: Option<String>, + pub stage: u32, + pub keep_stage: Vec<u32>, + pub keep_stage_std: Vec<u32>, + pub src: PathBuf, + /// defaults to `config.toml` + pub config: PathBuf, + pub jobs: Option<u32>, + pub cmd: Subcommand, + pub incremental: bool, + pub dry_run: bool, + /// `None` if we shouldn't download CI compiler artifacts, or the commit to download if we should. + #[cfg(not(test))] + download_rustc_commit: Option<String>, + #[cfg(test)] + pub download_rustc_commit: Option<String>, + + pub deny_warnings: bool, + pub backtrace_on_ice: bool, + + // llvm codegen options + pub llvm_skip_rebuild: bool, + pub llvm_assertions: bool, + pub llvm_tests: bool, + pub llvm_plugins: bool, + pub llvm_optimize: bool, + pub llvm_thin_lto: bool, + pub llvm_release_debuginfo: bool, + pub llvm_version_check: bool, + pub llvm_static_stdcpp: bool, + /// `None` if `llvm_from_ci` is true and we haven't yet downloaded llvm. + #[cfg(not(test))] + llvm_link_shared: Cell<Option<bool>>, + #[cfg(test)] + pub llvm_link_shared: Cell<Option<bool>>, + pub llvm_clang_cl: Option<String>, + pub llvm_targets: Option<String>, + pub llvm_experimental_targets: Option<String>, + pub llvm_link_jobs: Option<u32>, + pub llvm_version_suffix: Option<String>, + pub llvm_use_linker: Option<String>, + pub llvm_allow_old_toolchain: bool, + pub llvm_polly: bool, + pub llvm_clang: bool, + pub llvm_from_ci: bool, + pub llvm_build_config: HashMap<String, String>, + + pub use_lld: bool, + pub lld_enabled: bool, + pub llvm_tools_enabled: bool, + + pub llvm_cflags: Option<String>, + pub llvm_cxxflags: Option<String>, + pub llvm_ldflags: Option<String>, + pub llvm_use_libcxx: bool, + + // rust codegen options + pub rust_optimize: bool, + pub rust_codegen_units: Option<u32>, + pub rust_codegen_units_std: Option<u32>, + pub rust_debug_assertions: bool, + pub rust_debug_assertions_std: bool, + pub rust_overflow_checks: bool, + pub rust_overflow_checks_std: bool, + pub rust_debug_logging: bool, + pub rust_debuginfo_level_rustc: u32, + pub rust_debuginfo_level_std: u32, + pub rust_debuginfo_level_tools: u32, + pub rust_debuginfo_level_tests: u32, + pub rust_split_debuginfo: SplitDebuginfo, + pub rust_rpath: bool, + pub rustc_parallel: bool, + pub rustc_default_linker: Option<String>, + pub rust_optimize_tests: bool, + pub rust_dist_src: bool, + pub rust_codegen_backends: Vec<Interned<String>>, + pub rust_verify_llvm_ir: bool, + pub rust_thin_lto_import_instr_limit: Option<u32>, + pub rust_remap_debuginfo: bool, + pub rust_new_symbol_mangling: Option<bool>, + pub rust_profile_use: Option<String>, + pub rust_profile_generate: Option<String>, + pub llvm_profile_use: Option<String>, + pub llvm_profile_generate: bool, + pub llvm_libunwind_default: Option<LlvmLibunwind>, + + pub build: TargetSelection, + pub hosts: Vec<TargetSelection>, + pub targets: Vec<TargetSelection>, + pub local_rebuild: bool, + pub jemalloc: bool, + pub control_flow_guard: bool, + + // dist misc + pub dist_sign_folder: Option<PathBuf>, + pub dist_upload_addr: Option<String>, + pub dist_compression_formats: Option<Vec<String>>, + + // libstd features + pub backtrace: bool, // support for RUST_BACKTRACE + + // misc + pub low_priority: bool, + pub channel: String, + pub description: Option<String>, + pub verbose_tests: bool, + pub save_toolstates: Option<PathBuf>, + pub print_step_timings: bool, + pub print_step_rusage: bool, + pub missing_tools: bool, + + // Fallback musl-root for all targets + pub musl_root: Option<PathBuf>, + pub prefix: Option<PathBuf>, + pub sysconfdir: Option<PathBuf>, + pub datadir: Option<PathBuf>, + pub docdir: Option<PathBuf>, + pub bindir: PathBuf, + pub libdir: Option<PathBuf>, + pub mandir: Option<PathBuf>, + pub codegen_tests: bool, + pub nodejs: Option<PathBuf>, + pub npm: Option<PathBuf>, + pub gdb: Option<PathBuf>, + pub python: Option<PathBuf>, + pub cargo_native_static: bool, + pub configure_args: Vec<String>, + + // These are either the stage0 downloaded binaries or the locally installed ones. + pub initial_cargo: PathBuf, + pub initial_rustc: PathBuf, + #[cfg(not(test))] + initial_rustfmt: RefCell<RustfmtState>, + #[cfg(test)] + pub initial_rustfmt: RefCell<RustfmtState>, + pub out: PathBuf, +} + +#[derive(Default, Deserialize)] +#[cfg_attr(test, derive(Clone))] +pub struct Stage0Metadata { + pub config: Stage0Config, + pub checksums_sha256: HashMap<String, String>, + pub rustfmt: Option<RustfmtMetadata>, +} +#[derive(Default, Deserialize)] +#[cfg_attr(test, derive(Clone))] +pub struct Stage0Config { + pub dist_server: String, + pub artifacts_server: String, + pub artifacts_with_llvm_assertions_server: String, + pub git_merge_commit_email: String, + pub nightly_branch: String, +} +#[derive(Default, Deserialize)] +#[cfg_attr(test, derive(Clone))] +pub struct RustfmtMetadata { + pub date: String, + pub version: String, +} + +#[derive(Clone, Debug)] +pub enum RustfmtState { + SystemToolchain(PathBuf), + Downloaded(PathBuf), + Unavailable, + LazyEvaluated, +} + +impl Default for RustfmtState { + fn default() -> Self { + RustfmtState::LazyEvaluated + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum LlvmLibunwind { + No, + InTree, + System, +} + +impl Default for LlvmLibunwind { + fn default() -> Self { + Self::No + } +} + +impl FromStr for LlvmLibunwind { + type Err = String; + + fn from_str(value: &str) -> Result<Self, Self::Err> { + match value { + "no" => Ok(Self::No), + "in-tree" => Ok(Self::InTree), + "system" => Ok(Self::System), + invalid => Err(format!("Invalid value '{}' for rust.llvm-libunwind config.", invalid)), + } + } +} + +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum SplitDebuginfo { + Packed, + Unpacked, + Off, +} + +impl Default for SplitDebuginfo { + fn default() -> Self { + SplitDebuginfo::Off + } +} + +impl std::str::FromStr for SplitDebuginfo { + type Err = (); + + fn from_str(s: &str) -> Result<Self, Self::Err> { + match s { + "packed" => Ok(SplitDebuginfo::Packed), + "unpacked" => Ok(SplitDebuginfo::Unpacked), + "off" => Ok(SplitDebuginfo::Off), + _ => Err(()), + } + } +} + +impl SplitDebuginfo { + /// Returns the default `-Csplit-debuginfo` value for the current target. See the comment for + /// `rust.split-debuginfo` in `config.toml.example`. + fn default_for_platform(target: &str) -> Self { + if target.contains("apple") { + SplitDebuginfo::Unpacked + } else if target.contains("windows") { + SplitDebuginfo::Packed + } else { + SplitDebuginfo::Off + } + } +} + +#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct TargetSelection { + pub triple: Interned<String>, + file: Option<Interned<String>>, +} + +impl TargetSelection { + pub fn from_user(selection: &str) -> Self { + let path = Path::new(selection); + + let (triple, file) = if path.exists() { + let triple = path + .file_stem() + .expect("Target specification file has no file stem") + .to_str() + .expect("Target specification file stem is not UTF-8"); + + (triple, Some(selection)) + } else { + (selection, None) + }; + + let triple = INTERNER.intern_str(triple); + let file = file.map(|f| INTERNER.intern_str(f)); + + Self { triple, file } + } + + pub fn rustc_target_arg(&self) -> &str { + self.file.as_ref().unwrap_or(&self.triple) + } + + pub fn contains(&self, needle: &str) -> bool { + self.triple.contains(needle) + } + + pub fn starts_with(&self, needle: &str) -> bool { + self.triple.starts_with(needle) + } + + pub fn ends_with(&self, needle: &str) -> bool { + self.triple.ends_with(needle) + } +} + +impl fmt::Display for TargetSelection { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.triple)?; + if let Some(file) = self.file { + write!(f, "({})", file)?; + } + Ok(()) + } +} + +impl fmt::Debug for TargetSelection { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self) + } +} + +impl PartialEq<&str> for TargetSelection { + fn eq(&self, other: &&str) -> bool { + self.triple == *other + } +} + +/// Per-target configuration stored in the global configuration structure. +#[derive(Default)] +#[cfg_attr(test, derive(Clone))] +pub struct Target { + /// Some(path to llvm-config) if using an external LLVM. + pub llvm_config: Option<PathBuf>, + /// Some(path to FileCheck) if one was specified. + pub llvm_filecheck: Option<PathBuf>, + pub llvm_libunwind: Option<LlvmLibunwind>, + pub cc: Option<PathBuf>, + pub cxx: Option<PathBuf>, + pub ar: Option<PathBuf>, + pub ranlib: Option<PathBuf>, + pub default_linker: Option<PathBuf>, + pub linker: Option<PathBuf>, + pub ndk: Option<PathBuf>, + pub sanitizers: Option<bool>, + pub profiler: Option<bool>, + pub crt_static: Option<bool>, + pub musl_root: Option<PathBuf>, + pub musl_libdir: Option<PathBuf>, + pub wasi_root: Option<PathBuf>, + pub qemu_rootfs: Option<PathBuf>, + pub no_std: bool, +} + +impl Target { + pub fn from_triple(triple: &str) -> Self { + let mut target: Self = Default::default(); + if triple.contains("-none") + || triple.contains("nvptx") + || triple.contains("switch") + || triple.contains("-uefi") + { + target.no_std = true; + } + target + } +} +/// Structure of the `config.toml` file that configuration is read from. +/// +/// This structure uses `Decodable` to automatically decode a TOML configuration +/// file into this format, and then this is traversed and written into the above +/// `Config` structure. +#[derive(Deserialize, Default)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +struct TomlConfig { + changelog_seen: Option<usize>, + build: Option<Build>, + install: Option<Install>, + llvm: Option<Llvm>, + rust: Option<Rust>, + target: Option<HashMap<String, TomlTarget>>, + dist: Option<Dist>, + profile: Option<String>, +} + +trait Merge { + fn merge(&mut self, other: Self); +} + +impl Merge for TomlConfig { + fn merge( + &mut self, + TomlConfig { build, install, llvm, rust, dist, target, profile: _, changelog_seen: _ }: Self, + ) { + fn do_merge<T: Merge>(x: &mut Option<T>, y: Option<T>) { + if let Some(new) = y { + if let Some(original) = x { + original.merge(new); + } else { + *x = Some(new); + } + } + } + do_merge(&mut self.build, build); + do_merge(&mut self.install, install); + do_merge(&mut self.llvm, llvm); + do_merge(&mut self.rust, rust); + do_merge(&mut self.dist, dist); + assert!(target.is_none(), "merging target-specific config is not currently supported"); + } +} + +// We are using a decl macro instead of a derive proc macro here to reduce the compile time of +// rustbuild. +macro_rules! define_config { + ($(#[$attr:meta])* struct $name:ident { + $($field:ident: Option<$field_ty:ty> = $field_key:literal,)* + }) => { + $(#[$attr])* + struct $name { + $($field: Option<$field_ty>,)* + } + + impl Merge for $name { + fn merge(&mut self, other: Self) { + $( + if !self.$field.is_some() { + self.$field = other.$field; + } + )* + } + } + + // The following is a trimmed version of what serde_derive generates. All parts not relevant + // for toml deserialization have been removed. This reduces the binary size and improves + // compile time of rustbuild. + impl<'de> Deserialize<'de> for $name { + fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> + where + D: Deserializer<'de>, + { + struct Field; + impl<'de> serde::de::Visitor<'de> for Field { + type Value = $name; + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(concat!("struct ", stringify!($name))) + } + + #[inline] + fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error> + where + A: serde::de::MapAccess<'de>, + { + $(let mut $field: Option<$field_ty> = None;)* + while let Some(key) = + match serde::de::MapAccess::next_key::<String>(&mut map) { + Ok(val) => val, + Err(err) => { + return Err(err); + } + } + { + match &*key { + $($field_key => { + if $field.is_some() { + return Err(<A::Error as serde::de::Error>::duplicate_field( + $field_key, + )); + } + $field = match serde::de::MapAccess::next_value::<$field_ty>( + &mut map, + ) { + Ok(val) => Some(val), + Err(err) => { + return Err(err); + } + }; + })* + key => { + return Err(serde::de::Error::unknown_field(key, FIELDS)); + } + } + } + Ok($name { $($field),* }) + } + } + const FIELDS: &'static [&'static str] = &[ + $($field_key,)* + ]; + Deserializer::deserialize_struct( + deserializer, + stringify!($name), + FIELDS, + Field, + ) + } + } + } +} + +define_config! { + /// TOML representation of various global build decisions. + #[derive(Default)] + struct Build { + build: Option<String> = "build", + host: Option<Vec<String>> = "host", + target: Option<Vec<String>> = "target", + build_dir: Option<String> = "build-dir", + cargo: Option<String> = "cargo", + rustc: Option<String> = "rustc", + rustfmt: Option<PathBuf> = "rustfmt", + docs: Option<bool> = "docs", + compiler_docs: Option<bool> = "compiler-docs", + docs_minification: Option<bool> = "docs-minification", + submodules: Option<bool> = "submodules", + gdb: Option<String> = "gdb", + nodejs: Option<String> = "nodejs", + npm: Option<String> = "npm", + python: Option<String> = "python", + locked_deps: Option<bool> = "locked-deps", + vendor: Option<bool> = "vendor", + full_bootstrap: Option<bool> = "full-bootstrap", + extended: Option<bool> = "extended", + tools: Option<HashSet<String>> = "tools", + verbose: Option<usize> = "verbose", + sanitizers: Option<bool> = "sanitizers", + profiler: Option<bool> = "profiler", + cargo_native_static: Option<bool> = "cargo-native-static", + low_priority: Option<bool> = "low-priority", + configure_args: Option<Vec<String>> = "configure-args", + local_rebuild: Option<bool> = "local-rebuild", + print_step_timings: Option<bool> = "print-step-timings", + print_step_rusage: Option<bool> = "print-step-rusage", + check_stage: Option<u32> = "check-stage", + doc_stage: Option<u32> = "doc-stage", + build_stage: Option<u32> = "build-stage", + test_stage: Option<u32> = "test-stage", + install_stage: Option<u32> = "install-stage", + dist_stage: Option<u32> = "dist-stage", + bench_stage: Option<u32> = "bench-stage", + patch_binaries_for_nix: Option<bool> = "patch-binaries-for-nix", + metrics: Option<bool> = "metrics", + } +} + +define_config! { + /// TOML representation of various global install decisions. + struct Install { + prefix: Option<String> = "prefix", + sysconfdir: Option<String> = "sysconfdir", + docdir: Option<String> = "docdir", + bindir: Option<String> = "bindir", + libdir: Option<String> = "libdir", + mandir: Option<String> = "mandir", + datadir: Option<String> = "datadir", + } +} + +define_config! { + /// TOML representation of how the LLVM build is configured. + struct Llvm { + skip_rebuild: Option<bool> = "skip-rebuild", + optimize: Option<bool> = "optimize", + thin_lto: Option<bool> = "thin-lto", + release_debuginfo: Option<bool> = "release-debuginfo", + assertions: Option<bool> = "assertions", + tests: Option<bool> = "tests", + plugins: Option<bool> = "plugins", + ccache: Option<StringOrBool> = "ccache", + version_check: Option<bool> = "version-check", + static_libstdcpp: Option<bool> = "static-libstdcpp", + ninja: Option<bool> = "ninja", + targets: Option<String> = "targets", + experimental_targets: Option<String> = "experimental-targets", + link_jobs: Option<u32> = "link-jobs", + link_shared: Option<bool> = "link-shared", + version_suffix: Option<String> = "version-suffix", + clang_cl: Option<String> = "clang-cl", + cflags: Option<String> = "cflags", + cxxflags: Option<String> = "cxxflags", + ldflags: Option<String> = "ldflags", + use_libcxx: Option<bool> = "use-libcxx", + use_linker: Option<String> = "use-linker", + allow_old_toolchain: Option<bool> = "allow-old-toolchain", + polly: Option<bool> = "polly", + clang: Option<bool> = "clang", + download_ci_llvm: Option<StringOrBool> = "download-ci-llvm", + build_config: Option<HashMap<String, String>> = "build-config", + } +} + +define_config! { + struct Dist { + sign_folder: Option<String> = "sign-folder", + gpg_password_file: Option<String> = "gpg-password-file", + upload_addr: Option<String> = "upload-addr", + src_tarball: Option<bool> = "src-tarball", + missing_tools: Option<bool> = "missing-tools", + compression_formats: Option<Vec<String>> = "compression-formats", + } +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum StringOrBool { + String(String), + Bool(bool), +} + +impl Default for StringOrBool { + fn default() -> StringOrBool { + StringOrBool::Bool(false) + } +} + +define_config! { + /// TOML representation of how the Rust build is configured. + struct Rust { + optimize: Option<bool> = "optimize", + debug: Option<bool> = "debug", + codegen_units: Option<u32> = "codegen-units", + codegen_units_std: Option<u32> = "codegen-units-std", + debug_assertions: Option<bool> = "debug-assertions", + debug_assertions_std: Option<bool> = "debug-assertions-std", + overflow_checks: Option<bool> = "overflow-checks", + overflow_checks_std: Option<bool> = "overflow-checks-std", + debug_logging: Option<bool> = "debug-logging", + debuginfo_level: Option<u32> = "debuginfo-level", + debuginfo_level_rustc: Option<u32> = "debuginfo-level-rustc", + debuginfo_level_std: Option<u32> = "debuginfo-level-std", + debuginfo_level_tools: Option<u32> = "debuginfo-level-tools", + debuginfo_level_tests: Option<u32> = "debuginfo-level-tests", + split_debuginfo: Option<String> = "split-debuginfo", + run_dsymutil: Option<bool> = "run-dsymutil", + backtrace: Option<bool> = "backtrace", + incremental: Option<bool> = "incremental", + parallel_compiler: Option<bool> = "parallel-compiler", + default_linker: Option<String> = "default-linker", + channel: Option<String> = "channel", + description: Option<String> = "description", + musl_root: Option<String> = "musl-root", + rpath: Option<bool> = "rpath", + verbose_tests: Option<bool> = "verbose-tests", + optimize_tests: Option<bool> = "optimize-tests", + codegen_tests: Option<bool> = "codegen-tests", + ignore_git: Option<bool> = "ignore-git", + dist_src: Option<bool> = "dist-src", + save_toolstates: Option<String> = "save-toolstates", + codegen_backends: Option<Vec<String>> = "codegen-backends", + lld: Option<bool> = "lld", + use_lld: Option<bool> = "use-lld", + llvm_tools: Option<bool> = "llvm-tools", + deny_warnings: Option<bool> = "deny-warnings", + backtrace_on_ice: Option<bool> = "backtrace-on-ice", + verify_llvm_ir: Option<bool> = "verify-llvm-ir", + thin_lto_import_instr_limit: Option<u32> = "thin-lto-import-instr-limit", + remap_debuginfo: Option<bool> = "remap-debuginfo", + jemalloc: Option<bool> = "jemalloc", + test_compare_mode: Option<bool> = "test-compare-mode", + llvm_libunwind: Option<String> = "llvm-libunwind", + control_flow_guard: Option<bool> = "control-flow-guard", + new_symbol_mangling: Option<bool> = "new-symbol-mangling", + profile_generate: Option<String> = "profile-generate", + profile_use: Option<String> = "profile-use", + // ignored; this is set from an env var set by bootstrap.py + download_rustc: Option<StringOrBool> = "download-rustc", + } +} + +define_config! { + /// TOML representation of how each build target is configured. + struct TomlTarget { + cc: Option<String> = "cc", + cxx: Option<String> = "cxx", + ar: Option<String> = "ar", + ranlib: Option<String> = "ranlib", + default_linker: Option<PathBuf> = "default-linker", + linker: Option<String> = "linker", + llvm_config: Option<String> = "llvm-config", + llvm_filecheck: Option<String> = "llvm-filecheck", + llvm_libunwind: Option<String> = "llvm-libunwind", + android_ndk: Option<String> = "android-ndk", + sanitizers: Option<bool> = "sanitizers", + profiler: Option<bool> = "profiler", + crt_static: Option<bool> = "crt-static", + musl_root: Option<String> = "musl-root", + musl_libdir: Option<String> = "musl-libdir", + wasi_root: Option<String> = "wasi-root", + qemu_rootfs: Option<String> = "qemu-rootfs", + no_std: Option<bool> = "no-std", + } +} + +impl Config { + pub fn default_opts() -> Config { + let mut config = Config::default(); + config.llvm_optimize = true; + config.ninja_in_file = true; + config.llvm_version_check = true; + config.llvm_static_stdcpp = true; + config.backtrace = true; + config.rust_optimize = true; + config.rust_optimize_tests = true; + config.submodules = None; + config.docs = true; + config.docs_minification = true; + config.rust_rpath = true; + config.channel = "dev".to_string(); + config.codegen_tests = true; + config.rust_dist_src = true; + config.rust_codegen_backends = vec![INTERNER.intern_str("llvm")]; + config.deny_warnings = true; + config.bindir = "bin".into(); + + // set by build.rs + config.build = TargetSelection::from_user(&env!("BUILD_TRIPLE")); + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + // Undo `src/bootstrap` + config.src = manifest_dir.parent().unwrap().parent().unwrap().to_owned(); + config.out = PathBuf::from("build"); + + config.initial_cargo = PathBuf::from(env!("CARGO")); + config.initial_rustc = PathBuf::from(env!("RUSTC")); + + config + } + + pub fn parse(args: &[String]) -> Config { + let flags = Flags::parse(&args); + + let mut config = Config::default_opts(); + config.exclude = flags.exclude.into_iter().map(|path| TaskPath::parse(path)).collect(); + config.include_default_paths = flags.include_default_paths; + config.rustc_error_format = flags.rustc_error_format; + config.json_output = flags.json_output; + config.on_fail = flags.on_fail; + config.jobs = flags.jobs.map(threads_from_config); + config.cmd = flags.cmd; + config.incremental = flags.incremental; + config.dry_run = flags.dry_run; + config.keep_stage = flags.keep_stage; + config.keep_stage_std = flags.keep_stage_std; + config.color = flags.color; + if let Some(value) = flags.deny_warnings { + config.deny_warnings = value; + } + config.llvm_profile_use = flags.llvm_profile_use; + config.llvm_profile_generate = flags.llvm_profile_generate; + + let stage0_json = t!(std::fs::read(&config.src.join("src").join("stage0.json"))); + config.stage0_metadata = t!(serde_json::from_slice::<Stage0Metadata>(&stage0_json)); + + #[cfg(test)] + let get_toml = |_| TomlConfig::default(); + #[cfg(not(test))] + let get_toml = |file: &Path| { + let contents = + t!(fs::read_to_string(file), format!("config file {} not found", file.display())); + // Deserialize to Value and then TomlConfig to prevent the Deserialize impl of + // TomlConfig and sub types to be monomorphized 5x by toml. + match toml::from_str(&contents) + .and_then(|table: toml::Value| TomlConfig::deserialize(table)) + { + Ok(table) => table, + Err(err) => { + eprintln!("failed to parse TOML configuration '{}': {}", file.display(), err); + crate::detail_exit(2); + } + } + }; + + // Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory. + let toml_path = flags + .config + .clone() + .or_else(|| env::var_os("RUST_BOOTSTRAP_CONFIG").map(PathBuf::from)); + let using_default_path = toml_path.is_none(); + let mut toml_path = toml_path.unwrap_or_else(|| PathBuf::from("config.toml")); + if using_default_path && !toml_path.exists() { + toml_path = config.src.join(toml_path); + } + + // Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path, + // but not if `config.toml` hasn't been created. + let mut toml = if !using_default_path || toml_path.exists() { + get_toml(&toml_path) + } else { + TomlConfig::default() + }; + + if let Some(include) = &toml.profile { + let mut include_path = config.src.clone(); + include_path.push("src"); + include_path.push("bootstrap"); + include_path.push("defaults"); + include_path.push(format!("config.{}.toml", include)); + let included_toml = get_toml(&include_path); + toml.merge(included_toml); + } + + config.changelog_seen = toml.changelog_seen; + config.config = toml_path; + + let build = toml.build.unwrap_or_default(); + + set(&mut config.initial_rustc, build.rustc.map(PathBuf::from)); + set(&mut config.out, flags.build_dir.or_else(|| build.build_dir.map(PathBuf::from))); + // NOTE: Bootstrap spawns various commands with different working directories. + // To avoid writing to random places on the file system, `config.out` needs to be an absolute path. + if !config.out.is_absolute() { + // `canonicalize` requires the path to already exist. Use our vendored copy of `absolute` instead. + config.out = crate::util::absolute(&config.out); + } + + if config.dry_run { + let dir = config.out.join("tmp-dry-run"); + t!(fs::create_dir_all(&dir)); + config.out = dir; + } + + config.hosts = if let Some(arg_host) = flags.host { + arg_host + } else if let Some(file_host) = build.host { + file_host.iter().map(|h| TargetSelection::from_user(h)).collect() + } else { + vec![config.build] + }; + config.targets = if let Some(arg_target) = flags.target { + arg_target + } else if let Some(file_target) = build.target { + file_target.iter().map(|h| TargetSelection::from_user(h)).collect() + } else { + // If target is *not* configured, then default to the host + // toolchains. + config.hosts.clone() + }; + + config.nodejs = build.nodejs.map(PathBuf::from); + config.npm = build.npm.map(PathBuf::from); + config.gdb = build.gdb.map(PathBuf::from); + config.python = build.python.map(PathBuf::from); + config.submodules = build.submodules; + set(&mut config.low_priority, build.low_priority); + set(&mut config.compiler_docs, build.compiler_docs); + set(&mut config.docs_minification, build.docs_minification); + set(&mut config.docs, build.docs); + set(&mut config.locked_deps, build.locked_deps); + set(&mut config.vendor, build.vendor); + set(&mut config.full_bootstrap, build.full_bootstrap); + set(&mut config.extended, build.extended); + config.tools = build.tools; + set(&mut config.verbose, build.verbose); + set(&mut config.sanitizers, build.sanitizers); + set(&mut config.profiler, build.profiler); + set(&mut config.cargo_native_static, build.cargo_native_static); + set(&mut config.configure_args, build.configure_args); + set(&mut config.local_rebuild, build.local_rebuild); + set(&mut config.print_step_timings, build.print_step_timings); + set(&mut config.print_step_rusage, build.print_step_rusage); + set(&mut config.patch_binaries_for_nix, build.patch_binaries_for_nix); + + config.verbose = cmp::max(config.verbose, flags.verbose); + + if let Some(install) = toml.install { + config.prefix = install.prefix.map(PathBuf::from); + config.sysconfdir = install.sysconfdir.map(PathBuf::from); + config.datadir = install.datadir.map(PathBuf::from); + config.docdir = install.docdir.map(PathBuf::from); + set(&mut config.bindir, install.bindir.map(PathBuf::from)); + config.libdir = install.libdir.map(PathBuf::from); + config.mandir = install.mandir.map(PathBuf::from); + } + + // We want the llvm-skip-rebuild flag to take precedence over the + // skip-rebuild config.toml option so we store it separately + // so that we can infer the right value + let mut llvm_skip_rebuild = flags.llvm_skip_rebuild; + + // Store off these values as options because if they're not provided + // we'll infer default values for them later + let mut llvm_assertions = None; + let mut llvm_tests = None; + let mut llvm_plugins = None; + let mut debug = None; + let mut debug_assertions = None; + let mut debug_assertions_std = None; + let mut overflow_checks = None; + let mut overflow_checks_std = None; + let mut debug_logging = None; + let mut debuginfo_level = None; + let mut debuginfo_level_rustc = None; + let mut debuginfo_level_std = None; + let mut debuginfo_level_tools = None; + let mut debuginfo_level_tests = None; + let mut optimize = None; + let mut ignore_git = None; + + if let Some(llvm) = toml.llvm { + match llvm.ccache { + Some(StringOrBool::String(ref s)) => config.ccache = Some(s.to_string()), + Some(StringOrBool::Bool(true)) => { + config.ccache = Some("ccache".to_string()); + } + Some(StringOrBool::Bool(false)) | None => {} + } + set(&mut config.ninja_in_file, llvm.ninja); + llvm_assertions = llvm.assertions; + llvm_tests = llvm.tests; + llvm_plugins = llvm.plugins; + llvm_skip_rebuild = llvm_skip_rebuild.or(llvm.skip_rebuild); + set(&mut config.llvm_optimize, llvm.optimize); + set(&mut config.llvm_thin_lto, llvm.thin_lto); + set(&mut config.llvm_release_debuginfo, llvm.release_debuginfo); + set(&mut config.llvm_version_check, llvm.version_check); + set(&mut config.llvm_static_stdcpp, llvm.static_libstdcpp); + if let Some(v) = llvm.link_shared { + config.llvm_link_shared.set(Some(v)); + } + config.llvm_targets = llvm.targets.clone(); + config.llvm_experimental_targets = llvm.experimental_targets.clone(); + config.llvm_link_jobs = llvm.link_jobs; + config.llvm_version_suffix = llvm.version_suffix.clone(); + config.llvm_clang_cl = llvm.clang_cl.clone(); + + config.llvm_cflags = llvm.cflags.clone(); + config.llvm_cxxflags = llvm.cxxflags.clone(); + config.llvm_ldflags = llvm.ldflags.clone(); + set(&mut config.llvm_use_libcxx, llvm.use_libcxx); + config.llvm_use_linker = llvm.use_linker.clone(); + config.llvm_allow_old_toolchain = llvm.allow_old_toolchain.unwrap_or(false); + config.llvm_polly = llvm.polly.unwrap_or(false); + config.llvm_clang = llvm.clang.unwrap_or(false); + config.llvm_build_config = llvm.build_config.clone().unwrap_or(Default::default()); + config.llvm_from_ci = match llvm.download_ci_llvm { + Some(StringOrBool::String(s)) => { + assert!(s == "if-available", "unknown option `{}` for download-ci-llvm", s); + // This is currently all tier 1 targets and tier 2 targets with host tools + // (since others may not have CI artifacts) + // https://doc.rust-lang.org/rustc/platform-support.html#tier-1 + // FIXME: this is duplicated in bootstrap.py + let supported_platforms = [ + // tier 1 + "aarch64-unknown-linux-gnu", + "i686-pc-windows-gnu", + "i686-pc-windows-msvc", + "i686-unknown-linux-gnu", + "x86_64-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-gnu", + "x86_64-pc-windows-msvc", + // tier 2 with host tools + "aarch64-apple-darwin", + "aarch64-pc-windows-msvc", + "aarch64-unknown-linux-musl", + "arm-unknown-linux-gnueabi", + "arm-unknown-linux-gnueabihf", + "armv7-unknown-linux-gnueabihf", + "mips-unknown-linux-gnu", + "mips64-unknown-linux-gnuabi64", + "mips64el-unknown-linux-gnuabi64", + "mipsel-unknown-linux-gnu", + "powerpc-unknown-linux-gnu", + "powerpc64-unknown-linux-gnu", + "powerpc64le-unknown-linux-gnu", + "riscv64gc-unknown-linux-gnu", + "s390x-unknown-linux-gnu", + "x86_64-unknown-freebsd", + "x86_64-unknown-illumos", + "x86_64-unknown-linux-musl", + "x86_64-unknown-netbsd", + ]; + supported_platforms.contains(&&*config.build.triple) + } + Some(StringOrBool::Bool(b)) => b, + None => false, + }; + + if config.llvm_from_ci { + // None of the LLVM options, except assertions, are supported + // when using downloaded LLVM. We could just ignore these but + // that's potentially confusing, so force them to not be + // explicitly set. The defaults and CI defaults don't + // necessarily match but forcing people to match (somewhat + // arbitrary) CI configuration locally seems bad/hard. + check_ci_llvm!(llvm.optimize); + check_ci_llvm!(llvm.thin_lto); + check_ci_llvm!(llvm.release_debuginfo); + // CI-built LLVM can be either dynamic or static. We won't know until we download it. + check_ci_llvm!(llvm.link_shared); + check_ci_llvm!(llvm.static_libstdcpp); + check_ci_llvm!(llvm.targets); + check_ci_llvm!(llvm.experimental_targets); + check_ci_llvm!(llvm.link_jobs); + check_ci_llvm!(llvm.clang_cl); + check_ci_llvm!(llvm.version_suffix); + check_ci_llvm!(llvm.cflags); + check_ci_llvm!(llvm.cxxflags); + check_ci_llvm!(llvm.ldflags); + check_ci_llvm!(llvm.use_libcxx); + check_ci_llvm!(llvm.use_linker); + check_ci_llvm!(llvm.allow_old_toolchain); + check_ci_llvm!(llvm.polly); + check_ci_llvm!(llvm.clang); + check_ci_llvm!(llvm.build_config); + check_ci_llvm!(llvm.plugins); + } + + // NOTE: can never be hit when downloading from CI, since we call `check_ci_llvm!(thin_lto)` above. + if config.llvm_thin_lto && llvm.link_shared.is_none() { + // If we're building with ThinLTO on, by default we want to link + // to LLVM shared, to avoid re-doing ThinLTO (which happens in + // the link step) with each stage. + config.llvm_link_shared.set(Some(true)); + } + } + + if let Some(rust) = toml.rust { + debug = rust.debug; + debug_assertions = rust.debug_assertions; + debug_assertions_std = rust.debug_assertions_std; + overflow_checks = rust.overflow_checks; + overflow_checks_std = rust.overflow_checks_std; + debug_logging = rust.debug_logging; + debuginfo_level = rust.debuginfo_level; + debuginfo_level_rustc = rust.debuginfo_level_rustc; + debuginfo_level_std = rust.debuginfo_level_std; + debuginfo_level_tools = rust.debuginfo_level_tools; + debuginfo_level_tests = rust.debuginfo_level_tests; + config.rust_split_debuginfo = rust + .split_debuginfo + .as_deref() + .map(SplitDebuginfo::from_str) + .map(|v| v.expect("invalid value for rust.split_debuginfo")) + .unwrap_or(SplitDebuginfo::default_for_platform(&config.build.triple)); + optimize = rust.optimize; + ignore_git = rust.ignore_git; + config.rust_new_symbol_mangling = rust.new_symbol_mangling; + set(&mut config.rust_optimize_tests, rust.optimize_tests); + set(&mut config.codegen_tests, rust.codegen_tests); + set(&mut config.rust_rpath, rust.rpath); + set(&mut config.jemalloc, rust.jemalloc); + set(&mut config.test_compare_mode, rust.test_compare_mode); + set(&mut config.backtrace, rust.backtrace); + set(&mut config.channel, rust.channel); + config.description = rust.description; + set(&mut config.rust_dist_src, rust.dist_src); + set(&mut config.verbose_tests, rust.verbose_tests); + // in the case "false" is set explicitly, do not overwrite the command line args + if let Some(true) = rust.incremental { + config.incremental = true; + } + set(&mut config.use_lld, rust.use_lld); + set(&mut config.lld_enabled, rust.lld); + set(&mut config.llvm_tools_enabled, rust.llvm_tools); + config.rustc_parallel = rust.parallel_compiler.unwrap_or(false); + config.rustc_default_linker = rust.default_linker; + config.musl_root = rust.musl_root.map(PathBuf::from); + config.save_toolstates = rust.save_toolstates.map(PathBuf::from); + set(&mut config.deny_warnings, flags.deny_warnings.or(rust.deny_warnings)); + set(&mut config.backtrace_on_ice, rust.backtrace_on_ice); + set(&mut config.rust_verify_llvm_ir, rust.verify_llvm_ir); + config.rust_thin_lto_import_instr_limit = rust.thin_lto_import_instr_limit; + set(&mut config.rust_remap_debuginfo, rust.remap_debuginfo); + set(&mut config.control_flow_guard, rust.control_flow_guard); + config.llvm_libunwind_default = rust + .llvm_libunwind + .map(|v| v.parse().expect("failed to parse rust.llvm-libunwind")); + + if let Some(ref backends) = rust.codegen_backends { + config.rust_codegen_backends = + backends.iter().map(|s| INTERNER.intern_str(s)).collect(); + } + + config.rust_codegen_units = rust.codegen_units.map(threads_from_config); + config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config); + config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use); + config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate); + config.download_rustc_commit = download_ci_rustc_commit(&config, rust.download_rustc); + } else { + config.rust_profile_use = flags.rust_profile_use; + config.rust_profile_generate = flags.rust_profile_generate; + } + + if let Some(t) = toml.target { + for (triple, cfg) in t { + let mut target = Target::from_triple(&triple); + + if let Some(ref s) = cfg.llvm_config { + target.llvm_config = Some(config.src.join(s)); + } + if let Some(ref s) = cfg.llvm_filecheck { + target.llvm_filecheck = Some(config.src.join(s)); + } + target.llvm_libunwind = cfg + .llvm_libunwind + .as_ref() + .map(|v| v.parse().expect("failed to parse rust.llvm-libunwind")); + if let Some(ref s) = cfg.android_ndk { + target.ndk = Some(config.src.join(s)); + } + if let Some(s) = cfg.no_std { + target.no_std = s; + } + target.cc = cfg.cc.map(PathBuf::from); + target.cxx = cfg.cxx.map(PathBuf::from); + target.ar = cfg.ar.map(PathBuf::from); + target.ranlib = cfg.ranlib.map(PathBuf::from); + target.linker = cfg.linker.map(PathBuf::from); + target.crt_static = cfg.crt_static; + target.musl_root = cfg.musl_root.map(PathBuf::from); + target.musl_libdir = cfg.musl_libdir.map(PathBuf::from); + target.wasi_root = cfg.wasi_root.map(PathBuf::from); + target.qemu_rootfs = cfg.qemu_rootfs.map(PathBuf::from); + target.sanitizers = cfg.sanitizers; + target.profiler = cfg.profiler; + + config.target_config.insert(TargetSelection::from_user(&triple), target); + } + } + + if config.llvm_from_ci { + let triple = &config.build.triple; + let mut build_target = config + .target_config + .entry(config.build) + .or_insert_with(|| Target::from_triple(&triple)); + + check_ci_llvm!(build_target.llvm_config); + check_ci_llvm!(build_target.llvm_filecheck); + let ci_llvm_bin = config.out.join(&*config.build.triple).join("ci-llvm/bin"); + build_target.llvm_config = Some(ci_llvm_bin.join(exe("llvm-config", config.build))); + build_target.llvm_filecheck = Some(ci_llvm_bin.join(exe("FileCheck", config.build))); + } + + if let Some(t) = toml.dist { + config.dist_sign_folder = t.sign_folder.map(PathBuf::from); + config.dist_upload_addr = t.upload_addr; + config.dist_compression_formats = t.compression_formats; + set(&mut config.rust_dist_src, t.src_tarball); + set(&mut config.missing_tools, t.missing_tools); + } + + if let Some(r) = build.rustfmt { + *config.initial_rustfmt.borrow_mut() = if r.exists() { + RustfmtState::SystemToolchain(r) + } else { + RustfmtState::Unavailable + }; + } else { + // If using a system toolchain for bootstrapping, see if that has rustfmt available. + let host = config.build; + let rustfmt_path = config.initial_rustc.with_file_name(exe("rustfmt", host)); + let bin_root = config.out.join(host.triple).join("stage0"); + if !rustfmt_path.starts_with(&bin_root) { + // Using a system-provided toolchain; we shouldn't download rustfmt. + *config.initial_rustfmt.borrow_mut() = RustfmtState::SystemToolchain(rustfmt_path); + } + } + + // Now that we've reached the end of our configuration, infer the + // default values for all options that we haven't otherwise stored yet. + + config.llvm_skip_rebuild = llvm_skip_rebuild.unwrap_or(false); + config.llvm_assertions = llvm_assertions.unwrap_or(false); + config.llvm_tests = llvm_tests.unwrap_or(false); + config.llvm_plugins = llvm_plugins.unwrap_or(false); + config.rust_optimize = optimize.unwrap_or(true); + + let default = debug == Some(true); + config.rust_debug_assertions = debug_assertions.unwrap_or(default); + config.rust_debug_assertions_std = + debug_assertions_std.unwrap_or(config.rust_debug_assertions); + config.rust_overflow_checks = overflow_checks.unwrap_or(default); + config.rust_overflow_checks_std = + overflow_checks_std.unwrap_or(config.rust_overflow_checks); + + config.rust_debug_logging = debug_logging.unwrap_or(config.rust_debug_assertions); + + let with_defaults = |debuginfo_level_specific: Option<u32>| { + debuginfo_level_specific.or(debuginfo_level).unwrap_or(if debug == Some(true) { + 1 + } else { + 0 + }) + }; + config.rust_debuginfo_level_rustc = with_defaults(debuginfo_level_rustc); + config.rust_debuginfo_level_std = with_defaults(debuginfo_level_std); + config.rust_debuginfo_level_tools = with_defaults(debuginfo_level_tools); + config.rust_debuginfo_level_tests = debuginfo_level_tests.unwrap_or(0); + + let default = config.channel == "dev"; + config.ignore_git = ignore_git.unwrap_or(default); + + let download_rustc = config.download_rustc_commit.is_some(); + // See https://github.com/rust-lang/compiler-team/issues/326 + config.stage = match config.cmd { + Subcommand::Check { .. } => flags.stage.or(build.check_stage).unwrap_or(0), + // `download-rustc` only has a speed-up for stage2 builds. Default to stage2 unless explicitly overridden. + Subcommand::Doc { .. } => { + flags.stage.or(build.doc_stage).unwrap_or(if download_rustc { 2 } else { 0 }) + } + Subcommand::Build { .. } => { + flags.stage.or(build.build_stage).unwrap_or(if download_rustc { 2 } else { 1 }) + } + Subcommand::Test { .. } => { + flags.stage.or(build.test_stage).unwrap_or(if download_rustc { 2 } else { 1 }) + } + Subcommand::Bench { .. } => flags.stage.or(build.bench_stage).unwrap_or(2), + Subcommand::Dist { .. } => flags.stage.or(build.dist_stage).unwrap_or(2), + Subcommand::Install { .. } => flags.stage.or(build.install_stage).unwrap_or(2), + // These are all bootstrap tools, which don't depend on the compiler. + // The stage we pass shouldn't matter, but use 0 just in case. + Subcommand::Clean { .. } + | Subcommand::Clippy { .. } + | Subcommand::Fix { .. } + | Subcommand::Run { .. } + | Subcommand::Setup { .. } + | Subcommand::Format { .. } => flags.stage.unwrap_or(0), + }; + + // CI should always run stage 2 builds, unless it specifically states otherwise + #[cfg(not(test))] + if flags.stage.is_none() && crate::CiEnv::current() != crate::CiEnv::None { + match config.cmd { + Subcommand::Test { .. } + | Subcommand::Doc { .. } + | Subcommand::Build { .. } + | Subcommand::Bench { .. } + | Subcommand::Dist { .. } + | Subcommand::Install { .. } => { + assert_eq!( + config.stage, 2, + "x.py should be run with `--stage 2` on CI, but was run with `--stage {}`", + config.stage, + ); + } + Subcommand::Clean { .. } + | Subcommand::Check { .. } + | Subcommand::Clippy { .. } + | Subcommand::Fix { .. } + | Subcommand::Run { .. } + | Subcommand::Setup { .. } + | Subcommand::Format { .. } => {} + } + } + + config + } + + /// A git invocation which runs inside the source directory. + /// + /// Use this rather than `Command::new("git")` in order to support out-of-tree builds. + pub(crate) fn git(&self) -> Command { + let mut git = Command::new("git"); + git.current_dir(&self.src); + git + } + + pub(crate) fn artifact_channel(&self, commit: &str) -> String { + let mut channel = self.git(); + channel.arg("show").arg(format!("{}:src/ci/channel", commit)); + let channel = output(&mut channel); + channel.trim().to_owned() + } + + /// Try to find the relative path of `bindir`, otherwise return it in full. + pub fn bindir_relative(&self) -> &Path { + let bindir = &self.bindir; + if bindir.is_absolute() { + // Try to make it relative to the prefix. + if let Some(prefix) = &self.prefix { + if let Ok(stripped) = bindir.strip_prefix(prefix) { + return stripped; + } + } + } + bindir + } + + /// Try to find the relative path of `libdir`. + pub fn libdir_relative(&self) -> Option<&Path> { + let libdir = self.libdir.as_ref()?; + if libdir.is_relative() { + Some(libdir) + } else { + // Try to make it relative to the prefix. + libdir.strip_prefix(self.prefix.as_ref()?).ok() + } + } + + /// The absolute path to the downloaded LLVM artifacts. + pub(crate) fn ci_llvm_root(&self) -> PathBuf { + assert!(self.llvm_from_ci); + self.out.join(&*self.build.triple).join("ci-llvm") + } + + /// Determine whether llvm should be linked dynamically. + /// + /// If `false`, llvm should be linked statically. + /// This is computed on demand since LLVM might have to first be downloaded from CI. + pub(crate) fn llvm_link_shared(builder: &Builder<'_>) -> bool { + let mut opt = builder.config.llvm_link_shared.get(); + if opt.is_none() && builder.config.dry_run { + // just assume static for now - dynamic linking isn't supported on all platforms + return false; + } + + let llvm_link_shared = *opt.get_or_insert_with(|| { + if builder.config.llvm_from_ci { + crate::native::maybe_download_ci_llvm(builder); + let ci_llvm = builder.config.ci_llvm_root(); + let link_type = t!( + std::fs::read_to_string(ci_llvm.join("link-type.txt")), + format!("CI llvm missing: {}", ci_llvm.display()) + ); + link_type == "dynamic" + } else { + // unclear how thought-through this default is, but it maintains compatibility with + // previous behavior + false + } + }); + builder.config.llvm_link_shared.set(opt); + llvm_link_shared + } + + /// Return whether we will use a downloaded, pre-compiled version of rustc, or just build from source. + pub(crate) fn download_rustc(builder: &Builder<'_>) -> bool { + static DOWNLOAD_RUSTC: OnceCell<bool> = OnceCell::new(); + if builder.config.dry_run && DOWNLOAD_RUSTC.get().is_none() { + // avoid trying to actually download the commit + return false; + } + + *DOWNLOAD_RUSTC.get_or_init(|| match &builder.config.download_rustc_commit { + None => false, + Some(commit) => { + download_ci_rustc(builder, commit); + true + } + }) + } + + pub(crate) fn initial_rustfmt(builder: &Builder<'_>) -> Option<PathBuf> { + match &mut *builder.config.initial_rustfmt.borrow_mut() { + RustfmtState::SystemToolchain(p) | RustfmtState::Downloaded(p) => Some(p.clone()), + RustfmtState::Unavailable => None, + r @ RustfmtState::LazyEvaluated => { + if builder.config.dry_run { + return Some(PathBuf::new()); + } + let path = maybe_download_rustfmt(builder); + *r = if let Some(p) = &path { + RustfmtState::Downloaded(p.clone()) + } else { + RustfmtState::Unavailable + }; + path + } + } + } + + pub fn verbose(&self) -> bool { + self.verbose > 0 + } + + pub fn sanitizers_enabled(&self, target: TargetSelection) -> bool { + self.target_config.get(&target).map(|t| t.sanitizers).flatten().unwrap_or(self.sanitizers) + } + + pub fn any_sanitizers_enabled(&self) -> bool { + self.target_config.values().any(|t| t.sanitizers == Some(true)) || self.sanitizers + } + + pub fn profiler_enabled(&self, target: TargetSelection) -> bool { + self.target_config.get(&target).map(|t| t.profiler).flatten().unwrap_or(self.profiler) + } + + pub fn any_profiler_enabled(&self) -> bool { + self.target_config.values().any(|t| t.profiler == Some(true)) || self.profiler + } + + pub fn llvm_enabled(&self) -> bool { + self.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) + } + + pub fn llvm_libunwind(&self, target: TargetSelection) -> LlvmLibunwind { + self.target_config + .get(&target) + .and_then(|t| t.llvm_libunwind) + .or(self.llvm_libunwind_default) + .unwrap_or(LlvmLibunwind::No) + } + + pub fn submodules(&self, rust_info: &GitInfo) -> bool { + self.submodules.unwrap_or(rust_info.is_git()) + } +} + +fn set<T>(field: &mut T, val: Option<T>) { + if let Some(v) = val { + *field = v; + } +} + +fn threads_from_config(v: u32) -> u32 { + match v { + 0 => num_cpus::get() as u32, + n => n, + } +} + +/// Returns the commit to download, or `None` if we shouldn't download CI artifacts. +fn download_ci_rustc_commit( + config: &Config, + download_rustc: Option<StringOrBool>, +) -> Option<String> { + // If `download-rustc` is not set, default to rebuilding. + let if_unchanged = match download_rustc { + None | Some(StringOrBool::Bool(false)) => return None, + Some(StringOrBool::Bool(true)) => false, + Some(StringOrBool::String(s)) if s == "if-unchanged" => true, + Some(StringOrBool::String(other)) => { + panic!("unrecognized option for download-rustc: {}", other) + } + }; + + // Handle running from a directory other than the top level + let top_level = output(config.git().args(&["rev-parse", "--show-toplevel"])); + let top_level = top_level.trim_end(); + let compiler = format!("{top_level}/compiler/"); + let library = format!("{top_level}/library/"); + + // Look for a version to compare to based on the current commit. + // Only commits merged by bors will have CI artifacts. + let merge_base = output( + config + .git() + .arg("rev-list") + .arg(format!("--author={}", config.stage0_metadata.config.git_merge_commit_email)) + .args(&["-n1", "--first-parent", "HEAD"]), + ); + let commit = merge_base.trim_end(); + if commit.is_empty() { + println!("error: could not find commit hash for downloading rustc"); + println!("help: maybe your repository history is too shallow?"); + println!("help: consider disabling `download-rustc`"); + println!("help: or fetch enough history to include one upstream commit"); + crate::detail_exit(1); + } + + // Warn if there were changes to the compiler or standard library since the ancestor commit. + let has_changes = !t!(config + .git() + .args(&["diff-index", "--quiet", &commit, "--", &compiler, &library]) + .status()) + .success(); + if has_changes { + if if_unchanged { + if config.verbose > 0 { + println!( + "warning: saw changes to compiler/ or library/ since {commit}; \ + ignoring `download-rustc`" + ); + } + return None; + } + println!( + "warning: `download-rustc` is enabled, but there are changes to \ + compiler/ or library/" + ); + } + + Some(commit.to_string()) +} + +fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option<PathBuf> { + let RustfmtMetadata { date, version } = builder.config.stage0_metadata.rustfmt.as_ref()?; + let channel = format!("{version}-{date}"); + + let host = builder.config.build; + let rustfmt_path = builder.config.initial_rustc.with_file_name(exe("rustfmt", host)); + let bin_root = builder.config.out.join(host.triple).join("stage0"); + let rustfmt_stamp = bin_root.join(".rustfmt-stamp"); + if rustfmt_path.exists() && !program_out_of_date(&rustfmt_stamp, &channel) { + return Some(rustfmt_path); + } + + let filename = format!("rustfmt-{version}-{build}.tar.xz", build = host.triple); + download_component(builder, DownloadSource::Dist, filename, "rustfmt-preview", &date, "stage0"); + + builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt")); + builder.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt")); + + builder.create(&rustfmt_stamp, &channel); + Some(rustfmt_path) +} + +fn download_ci_rustc(builder: &Builder<'_>, commit: &str) { + builder.verbose(&format!("using downloaded stage2 artifacts from CI (commit {commit})")); + let channel = builder.config.artifact_channel(commit); + let host = builder.config.build.triple; + let bin_root = builder.out.join(host).join("ci-rustc"); + let rustc_stamp = bin_root.join(".rustc-stamp"); + + if !bin_root.join("bin").join("rustc").exists() || program_out_of_date(&rustc_stamp, commit) { + if bin_root.exists() { + t!(fs::remove_dir_all(&bin_root)); + } + let filename = format!("rust-std-{channel}-{host}.tar.xz"); + let pattern = format!("rust-std-{host}"); + download_ci_component(builder, filename, &pattern, commit); + let filename = format!("rustc-{channel}-{host}.tar.xz"); + download_ci_component(builder, filename, "rustc", commit); + // download-rustc doesn't need its own cargo, it can just use beta's. + let filename = format!("rustc-dev-{channel}-{host}.tar.xz"); + download_ci_component(builder, filename, "rustc-dev", commit); + + builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustc")); + builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc")); + let lib_dir = bin_root.join("lib"); + for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) { + let lib = t!(lib); + if lib.path().extension() == Some(OsStr::new("so")) { + builder.fix_bin_or_dylib(&lib.path()); + } + } + t!(fs::write(rustc_stamp, commit)); + } +} + +pub(crate) enum DownloadSource { + CI, + Dist, +} + +/// Download a single component of a CI-built toolchain (not necessarily a published nightly). +// NOTE: intentionally takes an owned string to avoid downloading multiple times by accident +fn download_ci_component(builder: &Builder<'_>, filename: String, prefix: &str, commit: &str) { + download_component(builder, DownloadSource::CI, filename, prefix, commit, "ci-rustc") +} + +fn download_component( + builder: &Builder<'_>, + mode: DownloadSource, + filename: String, + prefix: &str, + key: &str, + destination: &str, +) { + let cache_dst = builder.out.join("cache"); + let cache_dir = cache_dst.join(key); + if !cache_dir.exists() { + t!(fs::create_dir_all(&cache_dir)); + } + + let bin_root = builder.out.join(builder.config.build.triple).join(destination); + let tarball = cache_dir.join(&filename); + let (base_url, url, should_verify) = match mode { + DownloadSource::CI => ( + builder.config.stage0_metadata.config.artifacts_server.clone(), + format!("{key}/{filename}"), + false, + ), + DownloadSource::Dist => { + let dist_server = env::var("RUSTUP_DIST_SERVER") + .unwrap_or(builder.config.stage0_metadata.config.dist_server.to_string()); + // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json + (dist_server, format!("dist/{key}/{filename}"), true) + } + }; + + // For the beta compiler, put special effort into ensuring the checksums are valid. + // FIXME: maybe we should do this for download-rustc as well? but it would be a pain to update + // this on each and every nightly ... + let checksum = if should_verify { + let error = format!( + "src/stage0.json doesn't contain a checksum for {url}. \ + Pre-built artifacts might not be available for this \ + target at this time, see https://doc.rust-lang.org/nightly\ + /rustc/platform-support.html for more information." + ); + let sha256 = builder.config.stage0_metadata.checksums_sha256.get(&url).expect(&error); + if tarball.exists() { + if builder.verify(&tarball, sha256) { + builder.unpack(&tarball, &bin_root, prefix); + return; + } else { + builder.verbose(&format!( + "ignoring cached file {} due to failed verification", + tarball.display() + )); + builder.remove(&tarball); + } + } + Some(sha256) + } else if tarball.exists() { + builder.unpack(&tarball, &bin_root, prefix); + return; + } else { + None + }; + + builder.download_component(&format!("{base_url}/{url}"), &tarball, ""); + if let Some(sha256) = checksum { + if !builder.verify(&tarball, sha256) { + panic!("failed to verify {}", tarball.display()); + } + } + + builder.unpack(&tarball, &bin_root, prefix); +} diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py new file mode 100755 index 000000000..6b139decb --- /dev/null +++ b/src/bootstrap/configure.py @@ -0,0 +1,493 @@ +#!/usr/bin/env python + +# ignore-tidy-linelength + +from __future__ import absolute_import, division, print_function +import sys +import os +rust_dir = os.path.dirname(os.path.abspath(__file__)) +rust_dir = os.path.dirname(rust_dir) +rust_dir = os.path.dirname(rust_dir) +sys.path.append(os.path.join(rust_dir, "src", "bootstrap")) +import bootstrap + + +class Option(object): + def __init__(self, name, rustbuild, desc, value): + self.name = name + self.rustbuild = rustbuild + self.desc = desc + self.value = value + + +options = [] + + +def o(*args): + options.append(Option(*args, value=False)) + + +def v(*args): + options.append(Option(*args, value=True)) + + +o("debug", "rust.debug", "enables debugging environment; does not affect optimization of bootstrapped code") +o("docs", "build.docs", "build standard library documentation") +o("compiler-docs", "build.compiler-docs", "build compiler documentation") +o("optimize-tests", "rust.optimize-tests", "build tests with optimizations") +o("verbose-tests", "rust.verbose-tests", "enable verbose output when running tests") +o("ccache", "llvm.ccache", "invoke gcc/clang via ccache to reuse object files between builds") +o("sccache", None, "invoke gcc/clang via sccache to reuse object files between builds") +o("local-rust", None, "use an installed rustc rather than downloading a snapshot") +v("local-rust-root", None, "set prefix for local rust binary") +o("local-rebuild", "build.local-rebuild", "assume local-rust matches the current version, for rebuilds; implies local-rust, and is implied if local-rust already matches the current version") +o("llvm-static-stdcpp", "llvm.static-libstdcpp", "statically link to libstdc++ for LLVM") +o("llvm-link-shared", "llvm.link-shared", "prefer shared linking to LLVM (llvm-config --link-shared)") +o("rpath", "rust.rpath", "build rpaths into rustc itself") +o("llvm-version-check", "llvm.version-check", "check if the LLVM version is supported, build anyway") +o("codegen-tests", "rust.codegen-tests", "run the src/test/codegen tests") +o("option-checking", None, "complain about unrecognized options in this configure script") +o("ninja", "llvm.ninja", "build LLVM using the Ninja generator (for MSVC, requires building in the correct environment)") +o("locked-deps", "build.locked-deps", "force Cargo.lock to be up to date") +o("vendor", "build.vendor", "enable usage of vendored Rust crates") +o("sanitizers", "build.sanitizers", "build the sanitizer runtimes (asan, lsan, msan, tsan, hwasan)") +o("dist-src", "rust.dist-src", "when building tarballs enables building a source tarball") +o("cargo-native-static", "build.cargo-native-static", "static native libraries in cargo") +o("profiler", "build.profiler", "build the profiler runtime") +o("full-tools", None, "enable all tools") +o("lld", "rust.lld", "build lld") +o("clang", "llvm.clang", "build clang") +o("missing-tools", "dist.missing-tools", "allow failures when building tools") +o("use-libcxx", "llvm.use-libcxx", "build LLVM with libc++") +o("control-flow-guard", "rust.control-flow-guard", "Enable Control Flow Guard") + +v("llvm-cflags", "llvm.cflags", "build LLVM with these extra compiler flags") +v("llvm-cxxflags", "llvm.cxxflags", "build LLVM with these extra compiler flags") +v("llvm-ldflags", "llvm.ldflags", "build LLVM with these extra linker flags") + +v("llvm-libunwind", "rust.llvm-libunwind", "use LLVM libunwind") + +# Optimization and debugging options. These may be overridden by the release +# channel, etc. +o("optimize-llvm", "llvm.optimize", "build optimized LLVM") +o("llvm-assertions", "llvm.assertions", "build LLVM with assertions") +o("llvm-plugins", "llvm.plugins", "build LLVM with plugin interface") +o("debug-assertions", "rust.debug-assertions", "build with debugging assertions") +o("debug-assertions-std", "rust.debug-assertions-std", "build the standard library with debugging assertions") +o("overflow-checks", "rust.overflow-checks", "build with overflow checks") +o("overflow-checks-std", "rust.overflow-checks-std", "build the standard library with overflow checks") +o("llvm-release-debuginfo", "llvm.release-debuginfo", "build LLVM with debugger metadata") +v("debuginfo-level", "rust.debuginfo-level", "debuginfo level for Rust code") +v("debuginfo-level-rustc", "rust.debuginfo-level-rustc", "debuginfo level for the compiler") +v("debuginfo-level-std", "rust.debuginfo-level-std", "debuginfo level for the standard library") +v("debuginfo-level-tools", "rust.debuginfo-level-tools", "debuginfo level for the tools") +v("debuginfo-level-tests", "rust.debuginfo-level-tests", "debuginfo level for the test suites run with compiletest") +v("save-toolstates", "rust.save-toolstates", "save build and test status of external tools into this file") + +v("prefix", "install.prefix", "set installation prefix") +v("localstatedir", "install.localstatedir", "local state directory") +v("datadir", "install.datadir", "install data") +v("sysconfdir", "install.sysconfdir", "install system configuration files") +v("infodir", "install.infodir", "install additional info") +v("libdir", "install.libdir", "install libraries") +v("mandir", "install.mandir", "install man pages in PATH") +v("docdir", "install.docdir", "install documentation in PATH") +v("bindir", "install.bindir", "install binaries") + +v("llvm-root", None, "set LLVM root") +v("llvm-config", None, "set path to llvm-config") +v("llvm-filecheck", None, "set path to LLVM's FileCheck utility") +v("python", "build.python", "set path to python") +v("android-cross-path", "target.arm-linux-androideabi.android-ndk", + "Android NDK standalone path (deprecated)") +v("i686-linux-android-ndk", "target.i686-linux-android.android-ndk", + "i686-linux-android NDK standalone path") +v("arm-linux-androideabi-ndk", "target.arm-linux-androideabi.android-ndk", + "arm-linux-androideabi NDK standalone path") +v("armv7-linux-androideabi-ndk", "target.armv7-linux-androideabi.android-ndk", + "armv7-linux-androideabi NDK standalone path") +v("thumbv7neon-linux-androideabi-ndk", "target.thumbv7neon-linux-androideabi.android-ndk", + "thumbv7neon-linux-androideabi NDK standalone path") +v("aarch64-linux-android-ndk", "target.aarch64-linux-android.android-ndk", + "aarch64-linux-android NDK standalone path") +v("x86_64-linux-android-ndk", "target.x86_64-linux-android.android-ndk", + "x86_64-linux-android NDK standalone path") +v("musl-root", "target.x86_64-unknown-linux-musl.musl-root", + "MUSL root installation directory (deprecated)") +v("musl-root-x86_64", "target.x86_64-unknown-linux-musl.musl-root", + "x86_64-unknown-linux-musl install directory") +v("musl-root-i586", "target.i586-unknown-linux-musl.musl-root", + "i586-unknown-linux-musl install directory") +v("musl-root-i686", "target.i686-unknown-linux-musl.musl-root", + "i686-unknown-linux-musl install directory") +v("musl-root-arm", "target.arm-unknown-linux-musleabi.musl-root", + "arm-unknown-linux-musleabi install directory") +v("musl-root-armhf", "target.arm-unknown-linux-musleabihf.musl-root", + "arm-unknown-linux-musleabihf install directory") +v("musl-root-armv5te", "target.armv5te-unknown-linux-musleabi.musl-root", + "armv5te-unknown-linux-musleabi install directory") +v("musl-root-armv7", "target.armv7-unknown-linux-musleabi.musl-root", + "armv7-unknown-linux-musleabi install directory") +v("musl-root-armv7hf", "target.armv7-unknown-linux-musleabihf.musl-root", + "armv7-unknown-linux-musleabihf install directory") +v("musl-root-aarch64", "target.aarch64-unknown-linux-musl.musl-root", + "aarch64-unknown-linux-musl install directory") +v("musl-root-mips", "target.mips-unknown-linux-musl.musl-root", + "mips-unknown-linux-musl install directory") +v("musl-root-mipsel", "target.mipsel-unknown-linux-musl.musl-root", + "mipsel-unknown-linux-musl install directory") +v("musl-root-mips64", "target.mips64-unknown-linux-muslabi64.musl-root", + "mips64-unknown-linux-muslabi64 install directory") +v("musl-root-mips64el", "target.mips64el-unknown-linux-muslabi64.musl-root", + "mips64el-unknown-linux-muslabi64 install directory") +v("qemu-armhf-rootfs", "target.arm-unknown-linux-gnueabihf.qemu-rootfs", + "rootfs in qemu testing, you probably don't want to use this") +v("qemu-aarch64-rootfs", "target.aarch64-unknown-linux-gnu.qemu-rootfs", + "rootfs in qemu testing, you probably don't want to use this") +v("qemu-riscv64-rootfs", "target.riscv64gc-unknown-linux-gnu.qemu-rootfs", + "rootfs in qemu testing, you probably don't want to use this") +v("experimental-targets", "llvm.experimental-targets", + "experimental LLVM targets to build") +v("release-channel", "rust.channel", "the name of the release channel to build") +v("release-description", "rust.description", "optional descriptive string for version output") +v("dist-compression-formats", None, + "comma-separated list of compression formats to use") + +# Used on systems where "cc" is unavailable +v("default-linker", "rust.default-linker", "the default linker") + +# Many of these are saved below during the "writing configuration" step +# (others are conditionally saved). +o("manage-submodules", "build.submodules", "let the build manage the git submodules") +o("full-bootstrap", "build.full-bootstrap", "build three compilers instead of two (not recommended except for testing reproducible builds)") +o("extended", "build.extended", "build an extended rust tool set") + +v("tools", None, "List of extended tools will be installed") +v("codegen-backends", None, "List of codegen backends to build") +v("build", "build.build", "GNUs ./configure syntax LLVM build triple") +v("host", None, "GNUs ./configure syntax LLVM host triples") +v("target", None, "GNUs ./configure syntax LLVM target triples") + +v("set", None, "set arbitrary key/value pairs in TOML configuration") + + +def p(msg): + print("configure: " + msg) + + +def err(msg): + print("configure: error: " + msg) + sys.exit(1) + + +if '--help' in sys.argv or '-h' in sys.argv: + print('Usage: ./configure [options]') + print('') + print('Options') + for option in options: + if 'android' in option.name: + # no one needs to know about these obscure options + continue + if option.value: + print('\t{:30} {}'.format('--{}=VAL'.format(option.name), option.desc)) + else: + print('\t{:30} {}'.format('--enable-{}'.format(option.name), option.desc)) + print('') + print('This configure script is a thin configuration shim over the true') + print('configuration system, `config.toml`. You can explore the comments') + print('in `config.toml.example` next to this configure script to see') + print('more information about what each option is. Additionally you can') + print('pass `--set` as an argument to set arbitrary key/value pairs') + print('in the TOML configuration if desired') + print('') + print('Also note that all options which take `--enable` can similarly') + print('be passed with `--disable-foo` to forcibly disable the option') + sys.exit(0) + +# Parse all command line arguments into one of these three lists, handling +# boolean and value-based options separately +unknown_args = [] +need_value_args = [] +known_args = {} + +p("processing command line") +i = 1 +while i < len(sys.argv): + arg = sys.argv[i] + i += 1 + if not arg.startswith('--'): + unknown_args.append(arg) + continue + + found = False + for option in options: + value = None + if option.value: + keyval = arg[2:].split('=', 1) + key = keyval[0] + if option.name != key: + continue + + if len(keyval) > 1: + value = keyval[1] + elif i < len(sys.argv): + value = sys.argv[i] + i += 1 + else: + need_value_args.append(arg) + continue + else: + if arg[2:] == 'enable-' + option.name: + value = True + elif arg[2:] == 'disable-' + option.name: + value = False + else: + continue + + found = True + if option.name not in known_args: + known_args[option.name] = [] + known_args[option.name].append((option, value)) + break + + if not found: + unknown_args.append(arg) +p("") + +# Note: here and a few other places, we use [-1] to apply the *last* value +# passed. But if option-checking is enabled, then the known_args loop will +# also assert that options are only passed once. +option_checking = ('option-checking' not in known_args + or known_args['option-checking'][-1][1]) +if option_checking: + if len(unknown_args) > 0: + err("Option '" + unknown_args[0] + "' is not recognized") + if len(need_value_args) > 0: + err("Option '{0}' needs a value ({0}=val)".format(need_value_args[0])) + +# Parse all known arguments into a configuration structure that reflects the +# TOML we're going to write out +config = {} + + +def build(): + if 'build' in known_args: + return known_args['build'][-1][1] + return bootstrap.default_build_triple(verbose=False) + + +def set(key, value): + if isinstance(value, list): + # Remove empty values, which value.split(',') tends to generate. + value = [v for v in value if v] + + s = "{:20} := {}".format(key, value) + if len(s) < 70: + p(s) + else: + p(s[:70] + " ...") + + arr = config + parts = key.split('.') + for i, part in enumerate(parts): + if i == len(parts) - 1: + arr[part] = value + else: + if part not in arr: + arr[part] = {} + arr = arr[part] + + +for key in known_args: + # The `set` option is special and can be passed a bunch of times + if key == 'set': + for option, value in known_args[key]: + keyval = value.split('=', 1) + if len(keyval) == 1 or keyval[1] == "true": + value = True + elif keyval[1] == "false": + value = False + else: + value = keyval[1] + set(keyval[0], value) + continue + + # Ensure each option is only passed once + arr = known_args[key] + if option_checking and len(arr) > 1: + err("Option '{}' provided more than once".format(key)) + option, value = arr[-1] + + # If we have a clear avenue to set our value in rustbuild, do so + if option.rustbuild is not None: + set(option.rustbuild, value) + continue + + # Otherwise we're a "special" option and need some extra handling, so do + # that here. + if option.name == 'sccache': + set('llvm.ccache', 'sccache') + elif option.name == 'local-rust': + for path in os.environ['PATH'].split(os.pathsep): + if os.path.exists(path + '/rustc'): + set('build.rustc', path + '/rustc') + break + for path in os.environ['PATH'].split(os.pathsep): + if os.path.exists(path + '/cargo'): + set('build.cargo', path + '/cargo') + break + elif option.name == 'local-rust-root': + set('build.rustc', value + '/bin/rustc') + set('build.cargo', value + '/bin/cargo') + elif option.name == 'llvm-root': + set('target.{}.llvm-config'.format(build()), value + '/bin/llvm-config') + elif option.name == 'llvm-config': + set('target.{}.llvm-config'.format(build()), value) + elif option.name == 'llvm-filecheck': + set('target.{}.llvm-filecheck'.format(build()), value) + elif option.name == 'tools': + set('build.tools', value.split(',')) + elif option.name == 'codegen-backends': + set('rust.codegen-backends', value.split(',')) + elif option.name == 'host': + set('build.host', value.split(',')) + elif option.name == 'target': + set('build.target', value.split(',')) + elif option.name == 'full-tools': + set('rust.codegen-backends', ['llvm']) + set('rust.lld', True) + set('rust.llvm-tools', True) + set('build.extended', True) + elif option.name == 'option-checking': + # this was handled above + pass + elif option.name == 'dist-compression-formats': + set('dist.compression-formats', value.split(',')) + else: + raise RuntimeError("unhandled option {}".format(option.name)) + +set('build.configure-args', sys.argv[1:]) + +# "Parse" the `config.toml.example` file into the various sections, and we'll +# use this as a template of a `config.toml` to write out which preserves +# all the various comments and whatnot. +# +# Note that the `target` section is handled separately as we'll duplicate it +# per configured target, so there's a bit of special handling for that here. +sections = {} +cur_section = None +sections[None] = [] +section_order = [None] +targets = {} + +for line in open(rust_dir + '/config.toml.example').read().split("\n"): + if line.startswith('['): + cur_section = line[1:-1] + if cur_section.startswith('target'): + cur_section = 'target' + elif '.' in cur_section: + raise RuntimeError("don't know how to deal with section: {}".format(cur_section)) + sections[cur_section] = [line] + section_order.append(cur_section) + else: + sections[cur_section].append(line) + +# Fill out the `targets` array by giving all configured targets a copy of the +# `target` section we just loaded from the example config +configured_targets = [build()] +if 'build' in config: + if 'host' in config['build']: + configured_targets += config['build']['host'] + if 'target' in config['build']: + configured_targets += config['build']['target'] +if 'target' in config: + for target in config['target']: + configured_targets.append(target) +for target in configured_targets: + targets[target] = sections['target'][:] + targets[target][0] = targets[target][0].replace("x86_64-unknown-linux-gnu", "'{}'".format(target)) + + +def is_number(value): + try: + float(value) + return True + except ValueError: + return False + + +# Here we walk through the constructed configuration we have from the parsed +# command line arguments. We then apply each piece of configuration by +# basically just doing a `sed` to change the various configuration line to what +# we've got configure. +def to_toml(value): + if isinstance(value, bool): + if value: + return "true" + else: + return "false" + elif isinstance(value, list): + return '[' + ', '.join(map(to_toml, value)) + ']' + elif isinstance(value, str): + # Don't put quotes around numeric values + if is_number(value): + return value + else: + return "'" + value + "'" + else: + raise RuntimeError('no toml') + + +def configure_section(lines, config): + for key in config: + value = config[key] + found = False + for i, line in enumerate(lines): + if not line.startswith('#' + key + ' = '): + continue + found = True + lines[i] = "{} = {}".format(key, to_toml(value)) + break + if not found: + # These are used by rpm, but aren't accepted by x.py. + # Give a warning that they're ignored, but not a hard error. + if key in ["infodir", "localstatedir"]: + print("warning: {} will be ignored".format(key)) + else: + raise RuntimeError("failed to find config line for {}".format(key)) + + +for section_key in config: + section_config = config[section_key] + if section_key not in sections: + raise RuntimeError("config key {} not in sections".format(section_key)) + + if section_key == 'target': + for target in section_config: + configure_section(targets[target], section_config[target]) + else: + configure_section(sections[section_key], section_config) + +# Now that we've built up our `config.toml`, write it all out in the same +# order that we read it in. +p("") +p("writing `config.toml` in current directory") +with bootstrap.output('config.toml') as f: + for section in section_order: + if section == 'target': + for target in targets: + for line in targets[target]: + f.write(line + "\n") + else: + for line in sections[section]: + f.write(line + "\n") + +with bootstrap.output('Makefile') as f: + contents = os.path.join(rust_dir, 'src', 'bootstrap', 'mk', 'Makefile.in') + contents = open(contents).read() + contents = contents.replace("$(CFG_SRC_DIR)", rust_dir + '/') + contents = contents.replace("$(CFG_PYTHON)", sys.executable) + f.write(contents) + +p("") +p("run `python {}/x.py --help`".format(rust_dir)) diff --git a/src/bootstrap/defaults/README.md b/src/bootstrap/defaults/README.md new file mode 100644 index 000000000..f5b96db1b --- /dev/null +++ b/src/bootstrap/defaults/README.md @@ -0,0 +1,12 @@ +# About bootstrap defaults + +These defaults are intended to be a good starting point for working with x.py, +with the understanding that no one set of defaults make sense for everyone. + +They are still experimental, and we'd appreciate your help improving them! +If you use a setting that's not in these defaults that you think +others would benefit from, please [file an issue] or make a PR with the changes. +Similarly, if one of these defaults doesn't match what you use personally, +please open an issue to get it changed. + +[file an issue]: https://github.com/rust-lang/rust/issues/new/choose diff --git a/src/bootstrap/defaults/config.codegen.toml b/src/bootstrap/defaults/config.codegen.toml new file mode 100644 index 000000000..088cbd105 --- /dev/null +++ b/src/bootstrap/defaults/config.codegen.toml @@ -0,0 +1,19 @@ +# These defaults are meant for contributors to the compiler who modify codegen or LLVM +[build] +# Contributors working on the compiler will probably expect compiler docs to be generated. +compiler-docs = true + +[llvm] +# This enables debug-assertions in LLVM, +# catching logic errors in codegen much earlier in the process. +assertions = true + +[rust] +# This enables `RUSTC_LOG=debug`, avoiding confusing situations +# where adding `debug!()` appears to do nothing. +# However, it makes running the compiler slightly slower. +debug-logging = true +# This greatly increases the speed of rebuilds, especially when there are only minor changes. However, it makes the initial build slightly slower. +incremental = true +# Print backtrace on internal compiler errors during bootstrap +backtrace-on-ice = true diff --git a/src/bootstrap/defaults/config.compiler.toml b/src/bootstrap/defaults/config.compiler.toml new file mode 100644 index 000000000..2f4ccb825 --- /dev/null +++ b/src/bootstrap/defaults/config.compiler.toml @@ -0,0 +1,18 @@ +# These defaults are meant for contributors to the compiler who do not modify codegen or LLVM +[build] +# Contributors working on the compiler will probably expect compiler docs to be generated. +compiler-docs = true + +[rust] +# This enables `RUSTC_LOG=debug`, avoiding confusing situations +# where adding `debug!()` appears to do nothing. +# However, it makes running the compiler slightly slower. +debug-logging = true +# This greatly increases the speed of rebuilds, especially when there are only minor changes. However, it makes the initial build slightly slower. +incremental = true +# Print backtrace on internal compiler errors during bootstrap +backtrace-on-ice = true + +[llvm] +# Will download LLVM from CI if available on your platform. +download-ci-llvm = "if-available" diff --git a/src/bootstrap/defaults/config.library.toml b/src/bootstrap/defaults/config.library.toml new file mode 100644 index 000000000..7bc054d3a --- /dev/null +++ b/src/bootstrap/defaults/config.library.toml @@ -0,0 +1,14 @@ +# These defaults are meant for contributors to the standard library and documentation. +[build] +# When building the standard library, you almost never want to build the compiler itself. +build-stage = 0 +test-stage = 0 +bench-stage = 0 + +[rust] +# This greatly increases the speed of rebuilds, especially when there are only minor changes. However, it makes the initial build slightly slower. +incremental = true + +[llvm] +# Will download LLVM from CI if available on your platform. +download-ci-llvm = "if-available" diff --git a/src/bootstrap/defaults/config.tools.toml b/src/bootstrap/defaults/config.tools.toml new file mode 100644 index 000000000..6b6625342 --- /dev/null +++ b/src/bootstrap/defaults/config.tools.toml @@ -0,0 +1,22 @@ +# These defaults are meant for contributors to tools which build on the +# compiler, but do not modify it directly. +[rust] +# This enables `RUSTC_LOG=debug`, avoiding confusing situations +# where adding `debug!()` appears to do nothing. +# However, it makes running the compiler slightly slower. +debug-logging = true +# This greatly increases the speed of rebuilds, especially when there are only minor changes. However, it makes the initial build slightly slower. +incremental = true +# Download rustc from CI instead of building it from source. +# This cuts compile times by almost 60x, but means you can't modify the compiler. +download-rustc = "if-unchanged" + +[build] +# Document with the in-tree rustdoc by default, since `download-rustc` makes it quick to compile. +doc-stage = 2 +# Contributors working on tools will probably expect compiler docs to be generated, so they can figure out how to use the API. +compiler-docs = true + +[llvm] +# Will download LLVM from CI if available on your platform. +download-ci-llvm = "if-available" diff --git a/src/bootstrap/defaults/config.user.toml b/src/bootstrap/defaults/config.user.toml new file mode 100644 index 000000000..6647061d8 --- /dev/null +++ b/src/bootstrap/defaults/config.user.toml @@ -0,0 +1,9 @@ +# These defaults are meant for users and distro maintainers building from source, without intending to make multiple changes. +[build] +# When compiling from source, you almost always want a full stage 2 build, +# which has all the latest optimizations from nightly. +build-stage = 2 +test-stage = 2 +doc-stage = 2 +# When compiling from source, you usually want all tools. +extended = true diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs new file mode 100644 index 000000000..6291b204e --- /dev/null +++ b/src/bootstrap/dist.rs @@ -0,0 +1,2157 @@ +//! Implementation of the various distribution aspects of the compiler. +//! +//! This module is responsible for creating tarballs of the standard library, +//! compiler, and documentation. This ends up being what we distribute to +//! everyone as well. +//! +//! No tarball is actually created literally in this file, but rather we shell +//! out to `rust-installer` still. This may one day be replaced with bits and +//! pieces of `rustup.rs`! + +use std::collections::HashSet; +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; +use crate::cache::{Interned, INTERNER}; +use crate::compile; +use crate::config::TargetSelection; +use crate::tarball::{GeneratedTarball, OverlayKind, Tarball}; +use crate::tool::{self, Tool}; +use crate::util::{exe, is_dylib, output, t, timeit}; +use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS}; + +pub fn pkgname(builder: &Builder<'_>, component: &str) -> String { + format!("{}-{}", component, builder.rust_package_vers()) +} + +pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf { + builder.out.join("dist") +} + +pub fn tmpdir(builder: &Builder<'_>) -> PathBuf { + builder.out.join("tmp/dist") +} + +fn missing_tool(tool_name: &str, skip: bool) { + if skip { + println!("Unable to build {}, skipping dist", tool_name) + } else { + let help = "note: not all tools are available on all nightlies\nhelp: see https://forge.rust-lang.org/infra/toolstate.html for more information"; + panic!( + "Unable to build submodule tool {} (use `missing-tools = true` to ignore this failure)\n{}", + tool_name, help + ) + } +} + +fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool { + if !builder.config.extended { + return false; + } + builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool)) +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Docs { + pub host: TargetSelection, +} + +impl Step for Docs { + type Output = Option<GeneratedTarball>; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = run.builder.config.docs; + run.alias("rust-docs").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Docs { host: run.target }); + } + + /// Builds the `rust-docs` installer component. + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { + let host = self.host; + builder.default_doc(&[]); + + let dest = "share/doc/rust/html"; + + let mut tarball = Tarball::new(builder, "rust-docs", &host.triple); + tarball.set_product_name("Rust Documentation"); + tarball.add_bulk_dir(&builder.doc_out(host), dest); + tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644); + Some(tarball.generate()) + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustcDocs { + pub host: TargetSelection, +} + +impl Step for RustcDocs { + type Output = Option<GeneratedTarball>; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.alias("rustc-docs").default_condition(builder.config.compiler_docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustcDocs { host: run.target }); + } + + /// Builds the `rustc-docs` installer component. + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { + let host = self.host; + builder.default_doc(&[]); + + let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple); + tarball.set_product_name("Rustc Documentation"); + tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc"); + Some(tarball.generate()) + } +} + +fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> { + let mut found = Vec::with_capacity(files.len()); + + for file in files { + let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists()); + + if let Some(file_path) = file_path { + found.push(file_path); + } else { + panic!("Could not find '{}' in {:?}", file, path); + } + } + + found +} + +fn make_win_dist( + rust_root: &Path, + plat_root: &Path, + target: TargetSelection, + builder: &Builder<'_>, +) { + //Ask gcc where it keeps its stuff + let mut cmd = Command::new(builder.cc(target)); + cmd.arg("-print-search-dirs"); + let gcc_out = output(&mut cmd); + + let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect(); + let mut lib_path = Vec::new(); + + for line in gcc_out.lines() { + let idx = line.find(':').unwrap(); + let key = &line[..idx]; + let trim_chars: &[_] = &[' ', '=']; + let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars)); + + if key == "programs" { + bin_path.extend(value); + } else if key == "libraries" { + lib_path.extend(value); + } + } + + let compiler = if target == "i686-pc-windows-gnu" { + "i686-w64-mingw32-gcc.exe" + } else if target == "x86_64-pc-windows-gnu" { + "x86_64-w64-mingw32-gcc.exe" + } else { + "gcc.exe" + }; + let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"]; + let mut rustc_dlls = vec!["libwinpthread-1.dll"]; + if target.starts_with("i686-") { + rustc_dlls.push("libgcc_s_dw2-1.dll"); + } else { + rustc_dlls.push("libgcc_s_seh-1.dll"); + } + + let target_libs = [ + //MinGW libs + "libgcc.a", + "libgcc_eh.a", + "libgcc_s.a", + "libm.a", + "libmingw32.a", + "libmingwex.a", + "libstdc++.a", + "libiconv.a", + "libmoldname.a", + "libpthread.a", + //Windows import libs + "libadvapi32.a", + "libbcrypt.a", + "libcomctl32.a", + "libcomdlg32.a", + "libcredui.a", + "libcrypt32.a", + "libdbghelp.a", + "libgdi32.a", + "libimagehlp.a", + "libiphlpapi.a", + "libkernel32.a", + "libmsimg32.a", + "libmsvcrt.a", + "libodbc32.a", + "libole32.a", + "liboleaut32.a", + "libopengl32.a", + "libpsapi.a", + "librpcrt4.a", + "libsecur32.a", + "libsetupapi.a", + "libshell32.a", + "libsynchronization.a", + "libuser32.a", + "libuserenv.a", + "libuuid.a", + "libwinhttp.a", + "libwinmm.a", + "libwinspool.a", + "libws2_32.a", + "libwsock32.a", + ]; + + //Find mingw artifacts we want to bundle + let target_tools = find_files(&target_tools, &bin_path); + let rustc_dlls = find_files(&rustc_dlls, &bin_path); + let target_libs = find_files(&target_libs, &lib_path); + + // Copy runtime dlls next to rustc.exe + let dist_bin_dir = rust_root.join("bin/"); + fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed"); + for src in rustc_dlls { + builder.copy_to_folder(&src, &dist_bin_dir); + } + + //Copy platform tools to platform-specific bin directory + let target_bin_dir = plat_root + .join("lib") + .join("rustlib") + .join(target.triple) + .join("bin") + .join("self-contained"); + fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed"); + for src in target_tools { + builder.copy_to_folder(&src, &target_bin_dir); + } + + // Warn windows-gnu users that the bundled GCC cannot compile C files + builder.create( + &target_bin_dir.join("GCC-WARNING.txt"), + "gcc.exe contained in this folder cannot be used for compiling C files - it is only \ + used as a linker. In order to be able to compile projects containing C code use \ + the GCC provided by MinGW or Cygwin.", + ); + + //Copy platform libs to platform-specific lib directory + let target_lib_dir = plat_root + .join("lib") + .join("rustlib") + .join(target.triple) + .join("lib") + .join("self-contained"); + fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed"); + for src in target_libs { + builder.copy_to_folder(&src, &target_lib_dir); + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Mingw { + pub host: TargetSelection, +} + +impl Step for Mingw { + type Output = Option<GeneratedTarball>; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("rust-mingw") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Mingw { host: run.target }); + } + + /// Builds the `rust-mingw` installer component. + /// + /// This contains all the bits and pieces to run the MinGW Windows targets + /// without any extra installed software (e.g., we bundle gcc, libraries, etc). + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { + let host = self.host; + if !host.ends_with("pc-windows-gnu") { + return None; + } + + let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple); + tarball.set_product_name("Rust MinGW"); + + // The first argument is a "temporary directory" which is just + // thrown away (this contains the runtime DLLs included in the rustc package + // above) and the second argument is where to place all the MinGW components + // (which is what we want). + make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder); + + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Rustc { + pub compiler: Compiler, +} + +impl Step for Rustc { + type Output = GeneratedTarball; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("rustc") + } + + fn make_run(run: RunConfig<'_>) { + run.builder + .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) }); + } + + /// Creates the `rustc` installer component. + fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + let compiler = self.compiler; + let host = self.compiler.host; + + let tarball = Tarball::new(builder, "rustc", &host.triple); + + // Prepare the rustc "image", what will actually end up getting installed + prepare_image(builder, compiler, tarball.image_dir()); + + // On MinGW we've got a few runtime DLL dependencies that we need to + // include. The first argument to this script is where to put these DLLs + // (the image we're creating), and the second argument is a junk directory + // to ignore all other MinGW stuff the script creates. + // + // On 32-bit MinGW we're always including a DLL which needs some extra + // licenses to distribute. On 64-bit MinGW we don't actually distribute + // anything requiring us to distribute a license, but it's likely the + // install will *also* include the rust-mingw package, which also needs + // licenses, so to be safe we just include it here in all MinGW packages. + if host.ends_with("pc-windows-gnu") { + make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder); + tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc"); + } + + return tarball.generate(); + + fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) { + let host = compiler.host; + let src = builder.sysroot(compiler); + + // Copy rustc/rustdoc binaries + t!(fs::create_dir_all(image.join("bin"))); + builder.cp_r(&src.join("bin"), &image.join("bin")); + + builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755); + + let ra_proc_macro_srv = builder + .ensure(tool::RustAnalyzerProcMacroSrv { + compiler: builder.compiler_for( + compiler.stage, + builder.config.build, + compiler.host, + ), + target: compiler.host, + }) + .expect("rust-analyzer-proc-macro-server always builds"); + builder.install(&ra_proc_macro_srv, &image.join("libexec"), 0o755); + + let libdir_relative = builder.libdir_relative(compiler); + + // Copy runtime DLLs needed by the compiler + if libdir_relative.to_str() != Some("bin") { + let libdir = builder.rustc_libdir(compiler); + for entry in builder.read_dir(&libdir) { + let name = entry.file_name(); + if let Some(s) = name.to_str() { + if is_dylib(s) { + // Don't use custom libdir here because ^lib/ will be resolved again + // with installer + builder.install(&entry.path(), &image.join("lib"), 0o644); + } + } + } + } + + // Copy over the codegen backends + let backends_src = builder.sysroot_codegen_backends(compiler); + let backends_rel = backends_src + .strip_prefix(&src) + .unwrap() + .strip_prefix(builder.sysroot_libdir_relative(compiler)) + .unwrap(); + // Don't use custom libdir here because ^lib/ will be resolved again with installer + let backends_dst = image.join("lib").join(&backends_rel); + + t!(fs::create_dir_all(&backends_dst)); + builder.cp_r(&backends_src, &backends_dst); + + // Copy libLLVM.so to the lib dir as well, if needed. While not + // technically needed by rustc itself it's needed by lots of other + // components like the llvm tools and LLD. LLD is included below and + // tools/LLDB come later, so let's just throw it in the rustc + // component for now. + maybe_install_llvm_runtime(builder, host, image); + + let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin"); + t!(fs::create_dir_all(&dst_dir)); + + // Copy over lld if it's there + if builder.config.lld_enabled { + let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin"); + let rust_lld = exe("rust-lld", compiler.host); + builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld)); + // for `-Z gcc-ld=lld` + let gcc_lld_src_dir = src_dir.join("gcc-ld"); + let gcc_lld_dst_dir = dst_dir.join("gcc-ld"); + t!(fs::create_dir(&gcc_lld_dst_dir)); + let exe_name = exe("ld", compiler.host); + builder.copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name)); + } + + // Man pages + t!(fs::create_dir_all(image.join("share/man/man1"))); + let man_src = builder.src.join("src/doc/man"); + let man_dst = image.join("share/man/man1"); + + // don't use our `bootstrap::util::{copy, cp_r}`, because those try + // to hardlink, and we don't want to edit the source templates + for file_entry in builder.read_dir(&man_src) { + let page_src = file_entry.path(); + let page_dst = man_dst.join(file_entry.file_name()); + let src_text = t!(std::fs::read_to_string(&page_src)); + let new_text = src_text.replace("<INSERT VERSION HERE>", &builder.version); + t!(std::fs::write(&page_dst, &new_text)); + t!(fs::copy(&page_src, &page_dst)); + } + + // Debugger scripts + builder + .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host }); + + // Misc license info + let cp = |file: &str| { + builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644); + }; + cp("COPYRIGHT"); + cp("LICENSE-APACHE"); + cp("LICENSE-MIT"); + cp("README.md"); + } + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct DebuggerScripts { + pub sysroot: Interned<PathBuf>, + pub host: TargetSelection, +} + +impl Step for DebuggerScripts { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Copies debugger scripts for `target` into the `sysroot` specified. + fn run(self, builder: &Builder<'_>) { + let host = self.host; + let sysroot = self.sysroot; + let dst = sysroot.join("lib/rustlib/etc"); + t!(fs::create_dir_all(&dst)); + let cp_debugger_script = |file: &str| { + builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644); + }; + if host.contains("windows-msvc") { + // windbg debugger scripts + builder.install( + &builder.src.join("src/etc/rust-windbg.cmd"), + &sysroot.join("bin"), + 0o755, + ); + + cp_debugger_script("natvis/intrinsic.natvis"); + cp_debugger_script("natvis/liballoc.natvis"); + cp_debugger_script("natvis/libcore.natvis"); + cp_debugger_script("natvis/libstd.natvis"); + } else { + cp_debugger_script("rust_types.py"); + + // gdb debugger scripts + builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755); + builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755); + + cp_debugger_script("gdb_load_rust_pretty_printers.py"); + cp_debugger_script("gdb_lookup.py"); + cp_debugger_script("gdb_providers.py"); + + // lldb debugger scripts + builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755); + + cp_debugger_script("lldb_lookup.py"); + cp_debugger_script("lldb_providers.py"); + cp_debugger_script("lldb_commands") + } + } +} + +fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool { + // The only true set of target libraries came from the build triple, so + // let's reduce redundant work by only producing archives from that host. + if compiler.host != builder.config.build { + builder.info("\tskipping, not a build host"); + true + } else { + false + } +} + +/// Copy stamped files into an image's `target/lib` directory. +fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) { + let dst = image.join("lib/rustlib").join(target.triple).join("lib"); + let self_contained_dst = dst.join("self-contained"); + t!(fs::create_dir_all(&dst)); + t!(fs::create_dir_all(&self_contained_dst)); + for (path, dependency_type) in builder.read_stamp_file(stamp) { + if dependency_type == DependencyType::TargetSelfContained { + builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap())); + } else if dependency_type == DependencyType::Target || builder.config.build == target { + builder.copy(&path, &dst.join(path.file_name().unwrap())); + } + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Std { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Std { + type Output = Option<GeneratedTarball>; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("rust-std") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Std { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { + let compiler = self.compiler; + let target = self.target; + + if skip_host_target_lib(builder, compiler) { + return None; + } + + builder.ensure(compile::Std::new(compiler, target)); + + let mut tarball = Tarball::new(builder, "rust-std", &target.triple); + tarball.include_target_in_component_name(true); + + let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); + let stamp = compile::libstd_stamp(builder, compiler_to_use, target); + copy_target_libs(builder, target, &tarball.image_dir(), &stamp); + + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustcDev { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for RustcDev { + type Output = Option<GeneratedTarball>; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("rustc-dev") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustcDev { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { + let compiler = self.compiler; + let target = self.target; + if skip_host_target_lib(builder, compiler) { + return None; + } + + builder.ensure(compile::Rustc::new(compiler, target)); + + let tarball = Tarball::new(builder, "rustc-dev", &target.triple); + + let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); + let stamp = compile::librustc_stamp(builder, compiler_to_use, target); + copy_target_libs(builder, target, tarball.image_dir(), &stamp); + + let src_files = &["Cargo.lock"]; + // This is the reduced set of paths which will become the rustc-dev component + // (essentially the compiler crates and all of their path dependencies). + copy_src_dirs( + builder, + &builder.src, + &["compiler"], + &[], + &tarball.image_dir().join("lib/rustlib/rustc-src/rust"), + ); + for file in src_files { + tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644); + } + + Some(tarball.generate()) + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Analysis { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Analysis { + type Output = Option<GeneratedTarball>; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(&run.builder, "analysis"); + run.alias("rust-analysis").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Analysis { + // Find the actual compiler (handling the full bootstrap option) which + // produced the save-analysis data because that data isn't copied + // through the sysroot uplifting. + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + /// Creates a tarball of save-analysis metadata, if available. + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { + let compiler = self.compiler; + let target = self.target; + if compiler.host != builder.config.build { + return None; + } + + builder.ensure(compile::Std::new(compiler, target)); + let src = builder + .stage_out(compiler, Mode::Std) + .join(target.triple) + .join(builder.cargo_dir()) + .join("deps") + .join("save-analysis"); + + let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple); + tarball.include_target_in_component_name(true); + tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple)); + Some(tarball.generate()) + } +} + +/// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to +/// `dst_dir`. +fn copy_src_dirs( + builder: &Builder<'_>, + base: &Path, + src_dirs: &[&str], + exclude_dirs: &[&str], + dst_dir: &Path, +) { + fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool { + let spath = match path.to_str() { + Some(path) => path, + None => return false, + }; + if spath.ends_with('~') || spath.ends_with(".pyc") { + return false; + } + + const LLVM_PROJECTS: &[&str] = &[ + "llvm-project/clang", + "llvm-project\\clang", + "llvm-project/libunwind", + "llvm-project\\libunwind", + "llvm-project/lld", + "llvm-project\\lld", + "llvm-project/lldb", + "llvm-project\\lldb", + "llvm-project/llvm", + "llvm-project\\llvm", + "llvm-project/compiler-rt", + "llvm-project\\compiler-rt", + "llvm-project/cmake", + "llvm-project\\cmake", + ]; + if spath.contains("llvm-project") + && !spath.ends_with("llvm-project") + && !LLVM_PROJECTS.iter().any(|path| spath.contains(path)) + { + return false; + } + + const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"]; + if LLVM_TEST.iter().any(|path| spath.contains(path)) + && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s")) + { + return false; + } + + let full_path = Path::new(dir).join(path); + if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) { + return false; + } + + let excludes = [ + "CVS", + "RCS", + "SCCS", + ".git", + ".gitignore", + ".gitmodules", + ".gitattributes", + ".cvsignore", + ".svn", + ".arch-ids", + "{arch}", + "=RELEASE-ID", + "=meta-update", + "=update", + ".bzr", + ".bzrignore", + ".bzrtags", + ".hg", + ".hgignore", + ".hgrags", + "_darcs", + ]; + !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s)) + } + + // Copy the directories using our filter + for item in src_dirs { + let dst = &dst_dir.join(item); + t!(fs::create_dir_all(dst)); + builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path)); + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Src; + +impl Step for Src { + /// The output path of the src installer tarball + type Output = GeneratedTarball; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("rust-src") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Src); + } + + /// Creates the `rust-src` installer component + fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + let tarball = Tarball::new_targetless(builder, "rust-src"); + + // A lot of tools expect the rust-src component to be entirely in this directory, so if you + // change that (e.g. by adding another directory `lib/rustlib/src/foo` or + // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions + // and fix them... + // + // NOTE: if you update the paths here, you also should update the "virtual" path + // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs` + let dst_src = tarball.image_dir().join("lib/rustlib/src/rust"); + + let src_files = ["Cargo.lock"]; + // This is the reduced set of paths which will become the rust-src component + // (essentially libstd and all of its path dependencies). + copy_src_dirs( + builder, + &builder.src, + &["library", "src/llvm-project/libunwind"], + &[ + // not needed and contains symlinks which rustup currently + // chokes on when unpacking. + "library/backtrace/crates", + // these are 30MB combined and aren't necessary for building + // the standard library. + "library/stdarch/Cargo.toml", + "library/stdarch/crates/stdarch-verify", + "library/stdarch/crates/intrinsic-test", + ], + &dst_src, + ); + for file in src_files.iter() { + builder.copy(&builder.src.join(file), &dst_src.join(file)); + } + + tarball.generate() + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct PlainSourceTarball; + +impl Step for PlainSourceTarball { + /// Produces the location of the tarball generated + type Output = GeneratedTarball; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.alias("rustc-src").default_condition(builder.config.rust_dist_src) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(PlainSourceTarball); + } + + /// Creates the plain source tarball + fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which + // means neither rustup nor rustup-toolchain-install-master know how to download it. + // It also contains symbolic links, unlike other any other dist tarball. + // It's used for distros building rustc from source in a pre-vendored environment. + let mut tarball = Tarball::new(builder, "rustc", "src"); + tarball.permit_symlinks(true); + let plain_dst_src = tarball.image_dir(); + + // This is the set of root paths which will become part of the source package + let src_files = [ + "COPYRIGHT", + "LICENSE-APACHE", + "LICENSE-MIT", + "CONTRIBUTING.md", + "README.md", + "RELEASES.md", + "configure", + "x.py", + "config.toml.example", + "Cargo.toml", + "Cargo.lock", + ]; + let src_dirs = ["src", "compiler", "library"]; + + copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src); + + // Copy the files normally + for item in &src_files { + builder.copy(&builder.src.join(item), &plain_dst_src.join(item)); + } + + // Create the version file + builder.create(&plain_dst_src.join("version"), &builder.rust_version()); + if let Some(sha) = builder.rust_sha() { + builder.create(&plain_dst_src.join("git-commit-hash"), &sha); + } + + // If we're building from git sources, we need to vendor a complete distribution. + if builder.rust_info.is_git() { + // Ensure we have the submodules checked out. + builder.update_submodule(Path::new("src/tools/rust-analyzer")); + + // Vendor all Cargo dependencies + let mut cmd = Command::new(&builder.initial_cargo); + cmd.arg("vendor") + .arg("--sync") + .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml")) + .arg("--sync") + .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml")) + .arg("--sync") + .arg(builder.src.join("./src/bootstrap/Cargo.toml")) + .current_dir(&plain_dst_src); + + let config = if !builder.config.dry_run { + t!(String::from_utf8(t!(cmd.output()).stdout)) + } else { + String::new() + }; + + let cargo_config_dir = plain_dst_src.join(".cargo"); + builder.create_dir(&cargo_config_dir); + builder.create(&cargo_config_dir.join("config.toml"), &config); + } + + tarball.bare() + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Cargo { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Cargo { + type Output = Option<GeneratedTarball>; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(&run.builder, "cargo"); + run.alias("cargo").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Cargo { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { + let compiler = self.compiler; + let target = self.target; + + let cargo = builder.ensure(tool::Cargo { compiler, target }); + let src = builder.src.join("src/tools/cargo"); + let etc = src.join("src/etc"); + + // Prepare the image directory + let mut tarball = Tarball::new(builder, "cargo", &target.triple); + tarball.set_overlay(OverlayKind::Cargo); + + tarball.add_file(&cargo, "bin", 0o755); + tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644); + tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo"); + tarball.add_dir(etc.join("man"), "share/man/man1"); + tarball.add_legal_and_readme_to("share/doc/cargo"); + + for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") { + let dirent = dirent.expect("read dir entry"); + if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") { + tarball.add_file(&dirent.path(), "libexec", 0o755); + } + } + + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Rls { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Rls { + type Output = Option<GeneratedTarball>; + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(&run.builder, "rls"); + run.alias("rls").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Rls { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { + let compiler = self.compiler; + let target = self.target; + + let rls = builder + .ensure(tool::Rls { compiler, target, extra_features: Vec::new() }) + .or_else(|| { + missing_tool("RLS", builder.build.config.missing_tools); + None + })?; + + let mut tarball = Tarball::new(builder, "rls", &target.triple); + tarball.set_overlay(OverlayKind::RLS); + tarball.is_preview(true); + tarball.add_file(rls, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rls"); + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustAnalyzer { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for RustAnalyzer { + type Output = Option<GeneratedTarball>; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(&run.builder, "rust-analyzer"); + run.alias("rust-analyzer").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustAnalyzer { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { + let compiler = self.compiler; + let target = self.target; + + if target.contains("riscv64") { + // riscv64 currently has an LLVM bug that makes rust-analyzer unable + // to build. See #74813 for details. + return None; + } + + let rust_analyzer = builder + .ensure(tool::RustAnalyzer { compiler, target }) + .expect("rust-analyzer always builds"); + + let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple); + tarball.set_overlay(OverlayKind::RustAnalyzer); + tarball.is_preview(true); + tarball.add_file(rust_analyzer, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rust-analyzer"); + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Clippy { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Clippy { + type Output = Option<GeneratedTarball>; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(&run.builder, "clippy"); + run.alias("clippy").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Clippy { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { + let compiler = self.compiler; + let target = self.target; + + // Prepare the image directory + // We expect clippy to build, because we've exited this step above if tool + // state for clippy isn't testing. + let clippy = builder + .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() }) + .expect("clippy expected to build - essential tool"); + let cargoclippy = builder + .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() }) + .expect("clippy expected to build - essential tool"); + + let mut tarball = Tarball::new(builder, "clippy", &target.triple); + tarball.set_overlay(OverlayKind::Clippy); + tarball.is_preview(true); + tarball.add_file(clippy, "bin", 0o755); + tarball.add_file(cargoclippy, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/clippy"); + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Miri { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Miri { + type Output = Option<GeneratedTarball>; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(&run.builder, "miri"); + run.alias("miri").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Miri { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { + // This prevents miri from being built for "dist" or "install" + // on the stable/beta channels. It is a nightly-only tool and should + // not be included. + if !builder.build.unstable_features() { + return None; + } + let compiler = self.compiler; + let target = self.target; + + let miri = builder + .ensure(tool::Miri { compiler, target, extra_features: Vec::new() }) + .or_else(|| { + missing_tool("miri", builder.build.config.missing_tools); + None + })?; + let cargomiri = builder + .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() }) + .or_else(|| { + missing_tool("cargo miri", builder.build.config.missing_tools); + None + })?; + + let mut tarball = Tarball::new(builder, "miri", &target.triple); + tarball.set_overlay(OverlayKind::Miri); + tarball.is_preview(true); + tarball.add_file(miri, "bin", 0o755); + tarball.add_file(cargomiri, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/miri"); + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Rustfmt { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Rustfmt { + type Output = Option<GeneratedTarball>; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(&run.builder, "rustfmt"); + run.alias("rustfmt").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Rustfmt { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { + let compiler = self.compiler; + let target = self.target; + + let rustfmt = builder + .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() }) + .or_else(|| { + missing_tool("Rustfmt", builder.build.config.missing_tools); + None + })?; + let cargofmt = builder + .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() }) + .or_else(|| { + missing_tool("Cargofmt", builder.build.config.missing_tools); + None + })?; + + let mut tarball = Tarball::new(builder, "rustfmt", &target.triple); + tarball.set_overlay(OverlayKind::Rustfmt); + tarball.is_preview(true); + tarball.add_file(rustfmt, "bin", 0o755); + tarball.add_file(cargofmt, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rustfmt"); + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustDemangler { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for RustDemangler { + type Output = Option<GeneratedTarball>; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + // While other tools use `should_build_extended_tool` to decide whether to be run by + // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like + // the other ones or if `profiler = true`. Because we don't know the target at this stage + // we run the step by default when only `extended = true`, and decide whether to actually + // run it or not later. + let default = run.builder.config.extended; + run.alias("rust-demangler").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustDemangler { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { + let compiler = self.compiler; + let target = self.target; + + // Only build this extended tool if explicitly included in `tools`, or if `profiler = true` + let condition = should_build_extended_tool(builder, "rust-demangler") + || builder.config.profiler_enabled(target); + if builder.config.extended && !condition { + return None; + } + + let rust_demangler = builder + .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() }) + .expect("rust-demangler expected to build - in-tree tool"); + + // Prepare the image directory + let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple); + tarball.set_overlay(OverlayKind::RustDemangler); + tarball.is_preview(true); + tarball.add_file(&rust_demangler, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rust-demangler"); + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Extended { + stage: u32, + host: TargetSelection, + target: TargetSelection, +} + +impl Step for Extended { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.alias("extended").default_condition(builder.config.extended) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Extended { + stage: run.builder.top_stage, + host: run.builder.config.build, + target: run.target, + }); + } + + /// Creates a combined installer for the specified target in the provided stage. + fn run(self, builder: &Builder<'_>) { + let target = self.target; + let stage = self.stage; + let compiler = builder.compiler_for(self.stage, self.host, self.target); + + builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target)); + + let mut tarballs = Vec::new(); + let mut built_tools = HashSet::new(); + macro_rules! add_component { + ($name:expr => $step:expr) => { + if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) { + tarballs.push(tarball); + built_tools.insert($name); + } + }; + } + + // When rust-std package split from rustc, we needed to ensure that during + // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering + // the std files during uninstall. To do this ensure that rustc comes + // before rust-std in the list below. + tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) })); + tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std")); + + if target.ends_with("windows-gnu") { + tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw")); + } + + add_component!("rust-docs" => Docs { host: target }); + add_component!("rust-demangler"=> RustDemangler { compiler, target }); + add_component!("cargo" => Cargo { compiler, target }); + add_component!("rustfmt" => Rustfmt { compiler, target }); + add_component!("rls" => Rls { compiler, target }); + add_component!("rust-analyzer" => RustAnalyzer { compiler, target }); + add_component!("llvm-components" => LlvmTools { target }); + add_component!("clippy" => Clippy { compiler, target }); + add_component!("miri" => Miri { compiler, target }); + add_component!("analysis" => Analysis { compiler, target }); + + let etc = builder.src.join("src/etc/installer"); + + // Avoid producing tarballs during a dry run. + if builder.config.dry_run { + return; + } + + let tarball = Tarball::new(builder, "rust", &target.triple); + let generated = tarball.combine(&tarballs); + + let tmp = tmpdir(builder).join("combined-tarball"); + let work = generated.work_dir(); + + let mut license = String::new(); + license += &builder.read(&builder.src.join("COPYRIGHT")); + license += &builder.read(&builder.src.join("LICENSE-APACHE")); + license += &builder.read(&builder.src.join("LICENSE-MIT")); + license.push('\n'); + license.push('\n'); + + let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18"; + let mut rtf = rtf.to_string(); + rtf.push('\n'); + for line in license.lines() { + rtf.push_str(line); + rtf.push_str("\\line "); + } + rtf.push('}'); + + fn filter(contents: &str, marker: &str) -> String { + let start = format!("tool-{}-start", marker); + let end = format!("tool-{}-end", marker); + let mut lines = Vec::new(); + let mut omitted = false; + for line in contents.lines() { + if line.contains(&start) { + omitted = true; + } else if line.contains(&end) { + omitted = false; + } else if !omitted { + lines.push(line); + } + } + + lines.join("\n") + } + + let xform = |p: &Path| { + let mut contents = t!(fs::read_to_string(p)); + for tool in &["rust-demangler", "rls", "rust-analyzer", "miri", "rustfmt"] { + if !built_tools.contains(tool) { + contents = filter(&contents, tool); + } + } + let ret = tmp.join(p.file_name().unwrap()); + t!(fs::write(&ret, &contents)); + ret + }; + + if target.contains("apple-darwin") { + builder.info("building pkg installer"); + let pkg = tmp.join("pkg"); + let _ = fs::remove_dir_all(&pkg); + + let pkgbuild = |component: &str| { + let mut cmd = Command::new("pkgbuild"); + cmd.arg("--identifier") + .arg(format!("org.rust-lang.{}", component)) + .arg("--scripts") + .arg(pkg.join(component)) + .arg("--nopayload") + .arg(pkg.join(component).with_extension("pkg")); + builder.run(&mut cmd); + }; + + let prepare = |name: &str| { + builder.create_dir(&pkg.join(name)); + builder.cp_r( + &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)), + &pkg.join(name), + ); + builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755); + pkgbuild(name); + }; + prepare("rustc"); + prepare("cargo"); + prepare("rust-std"); + prepare("rust-analysis"); + prepare("clippy"); + for tool in &["rust-docs", "rust-demangler", "rls", "rust-analyzer", "miri"] { + if built_tools.contains(tool) { + prepare(tool); + } + } + // create an 'uninstall' package + builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755); + pkgbuild("uninstall"); + + builder.create_dir(&pkg.join("res")); + builder.create(&pkg.join("res/LICENSE.txt"), &license); + builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644); + let mut cmd = Command::new("productbuild"); + cmd.arg("--distribution") + .arg(xform(&etc.join("pkg/Distribution.xml"))) + .arg("--resources") + .arg(pkg.join("res")) + .arg(distdir(builder).join(format!( + "{}-{}.pkg", + pkgname(builder, "rust"), + target.triple + ))) + .arg("--package-path") + .arg(&pkg); + let _time = timeit(builder); + builder.run(&mut cmd); + } + + if target.contains("windows") { + let exe = tmp.join("exe"); + let _ = fs::remove_dir_all(&exe); + + let prepare = |name: &str| { + builder.create_dir(&exe.join(name)); + let dir = if name == "rust-std" || name == "rust-analysis" { + format!("{}-{}", name, target.triple) + } else if name == "rls" { + "rls-preview".to_string() + } else if name == "rust-analyzer" { + "rust-analyzer-preview".to_string() + } else if name == "clippy" { + "clippy-preview".to_string() + } else if name == "rust-demangler" { + "rust-demangler-preview".to_string() + } else if name == "miri" { + "miri-preview".to_string() + } else { + name.to_string() + }; + builder.cp_r( + &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir), + &exe.join(name), + ); + builder.remove(&exe.join(name).join("manifest.in")); + }; + prepare("rustc"); + prepare("cargo"); + prepare("rust-analysis"); + prepare("rust-docs"); + prepare("rust-std"); + prepare("clippy"); + for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] { + if built_tools.contains(tool) { + prepare(tool); + } + } + if target.ends_with("windows-gnu") { + prepare("rust-mingw"); + } + + builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644); + + // Generate msi installer + let wix_path = env::var_os("WIX") + .expect("`WIX` environment variable must be set for generating MSI installer(s)."); + let wix = PathBuf::from(wix_path); + let heat = wix.join("bin/heat.exe"); + let candle = wix.join("bin/candle.exe"); + let light = wix.join("bin/light.exe"); + + let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"]; + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rustc") + .args(&heat_flags) + .arg("-cg") + .arg("RustcGroup") + .arg("-dr") + .arg("Rustc") + .arg("-var") + .arg("var.RustcDir") + .arg("-out") + .arg(exe.join("RustcGroup.wxs")), + ); + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rust-docs") + .args(&heat_flags) + .arg("-cg") + .arg("DocsGroup") + .arg("-dr") + .arg("Docs") + .arg("-var") + .arg("var.DocsDir") + .arg("-out") + .arg(exe.join("DocsGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/squash-components.xsl")), + ); + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("cargo") + .args(&heat_flags) + .arg("-cg") + .arg("CargoGroup") + .arg("-dr") + .arg("Cargo") + .arg("-var") + .arg("var.CargoDir") + .arg("-out") + .arg(exe.join("CargoGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/remove-duplicates.xsl")), + ); + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rust-std") + .args(&heat_flags) + .arg("-cg") + .arg("StdGroup") + .arg("-dr") + .arg("Std") + .arg("-var") + .arg("var.StdDir") + .arg("-out") + .arg(exe.join("StdGroup.wxs")), + ); + if built_tools.contains("rls") { + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rls") + .args(&heat_flags) + .arg("-cg") + .arg("RlsGroup") + .arg("-dr") + .arg("Rls") + .arg("-var") + .arg("var.RlsDir") + .arg("-out") + .arg(exe.join("RlsGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/remove-duplicates.xsl")), + ); + } + if built_tools.contains("rust-analyzer") { + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rust-analyzer") + .args(&heat_flags) + .arg("-cg") + .arg("RustAnalyzerGroup") + .arg("-dr") + .arg("RustAnalyzer") + .arg("-var") + .arg("var.RustAnalyzerDir") + .arg("-out") + .arg(exe.join("RustAnalyzerGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/remove-duplicates.xsl")), + ); + } + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("clippy") + .args(&heat_flags) + .arg("-cg") + .arg("ClippyGroup") + .arg("-dr") + .arg("Clippy") + .arg("-var") + .arg("var.ClippyDir") + .arg("-out") + .arg(exe.join("ClippyGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/remove-duplicates.xsl")), + ); + if built_tools.contains("rust-demangler") { + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rust-demangler") + .args(&heat_flags) + .arg("-cg") + .arg("RustDemanglerGroup") + .arg("-dr") + .arg("RustDemangler") + .arg("-var") + .arg("var.RustDemanglerDir") + .arg("-out") + .arg(exe.join("RustDemanglerGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/remove-duplicates.xsl")), + ); + } + if built_tools.contains("miri") { + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("miri") + .args(&heat_flags) + .arg("-cg") + .arg("MiriGroup") + .arg("-dr") + .arg("Miri") + .arg("-var") + .arg("var.MiriDir") + .arg("-out") + .arg(exe.join("MiriGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/remove-duplicates.xsl")), + ); + } + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rust-analysis") + .args(&heat_flags) + .arg("-cg") + .arg("AnalysisGroup") + .arg("-dr") + .arg("Analysis") + .arg("-var") + .arg("var.AnalysisDir") + .arg("-out") + .arg(exe.join("AnalysisGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/remove-duplicates.xsl")), + ); + if target.ends_with("windows-gnu") { + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rust-mingw") + .args(&heat_flags) + .arg("-cg") + .arg("GccGroup") + .arg("-dr") + .arg("Gcc") + .arg("-var") + .arg("var.GccDir") + .arg("-out") + .arg(exe.join("GccGroup.wxs")), + ); + } + + let candle = |input: &Path| { + let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj"); + let arch = if target.contains("x86_64") { "x64" } else { "x86" }; + let mut cmd = Command::new(&candle); + cmd.current_dir(&exe) + .arg("-nologo") + .arg("-dRustcDir=rustc") + .arg("-dDocsDir=rust-docs") + .arg("-dCargoDir=cargo") + .arg("-dStdDir=rust-std") + .arg("-dAnalysisDir=rust-analysis") + .arg("-dClippyDir=clippy") + .arg("-arch") + .arg(&arch) + .arg("-out") + .arg(&output) + .arg(&input); + add_env(builder, &mut cmd, target); + + if built_tools.contains("rust-demangler") { + cmd.arg("-dRustDemanglerDir=rust-demangler"); + } + if built_tools.contains("rls") { + cmd.arg("-dRlsDir=rls"); + } + if built_tools.contains("rust-analyzer") { + cmd.arg("-dRustAnalyzerDir=rust-analyzer"); + } + if built_tools.contains("miri") { + cmd.arg("-dMiriDir=miri"); + } + if target.ends_with("windows-gnu") { + cmd.arg("-dGccDir=rust-mingw"); + } + builder.run(&mut cmd); + }; + candle(&xform(&etc.join("msi/rust.wxs"))); + candle(&etc.join("msi/ui.wxs")); + candle(&etc.join("msi/rustwelcomedlg.wxs")); + candle("RustcGroup.wxs".as_ref()); + candle("DocsGroup.wxs".as_ref()); + candle("CargoGroup.wxs".as_ref()); + candle("StdGroup.wxs".as_ref()); + candle("ClippyGroup.wxs".as_ref()); + if built_tools.contains("rust-demangler") { + candle("RustDemanglerGroup.wxs".as_ref()); + } + if built_tools.contains("rls") { + candle("RlsGroup.wxs".as_ref()); + } + if built_tools.contains("rust-analyzer") { + candle("RustAnalyzerGroup.wxs".as_ref()); + } + if built_tools.contains("miri") { + candle("MiriGroup.wxs".as_ref()); + } + candle("AnalysisGroup.wxs".as_ref()); + + if target.ends_with("windows-gnu") { + candle("GccGroup.wxs".as_ref()); + } + + builder.create(&exe.join("LICENSE.rtf"), &rtf); + builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644); + builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644); + + builder.info(&format!("building `msi` installer with {:?}", light)); + let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple); + let mut cmd = Command::new(&light); + cmd.arg("-nologo") + .arg("-ext") + .arg("WixUIExtension") + .arg("-ext") + .arg("WixUtilExtension") + .arg("-out") + .arg(exe.join(&filename)) + .arg("rust.wixobj") + .arg("ui.wixobj") + .arg("rustwelcomedlg.wixobj") + .arg("RustcGroup.wixobj") + .arg("DocsGroup.wixobj") + .arg("CargoGroup.wixobj") + .arg("StdGroup.wixobj") + .arg("AnalysisGroup.wixobj") + .arg("ClippyGroup.wixobj") + .current_dir(&exe); + + if built_tools.contains("rls") { + cmd.arg("RlsGroup.wixobj"); + } + if built_tools.contains("rust-analyzer") { + cmd.arg("RustAnalyzerGroup.wixobj"); + } + if built_tools.contains("rust-demangler") { + cmd.arg("RustDemanglerGroup.wixobj"); + } + if built_tools.contains("miri") { + cmd.arg("MiriGroup.wixobj"); + } + + if target.ends_with("windows-gnu") { + cmd.arg("GccGroup.wixobj"); + } + // ICE57 wrongly complains about the shortcuts + cmd.arg("-sice:ICE57"); + + let _time = timeit(builder); + builder.run(&mut cmd); + + if !builder.config.dry_run { + t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename))); + } + } + } +} + +fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) { + let mut parts = builder.version.split('.'); + cmd.env("CFG_RELEASE_INFO", builder.rust_version()) + .env("CFG_RELEASE_NUM", &builder.version) + .env("CFG_RELEASE", builder.rust_release()) + .env("CFG_VER_MAJOR", parts.next().unwrap()) + .env("CFG_VER_MINOR", parts.next().unwrap()) + .env("CFG_VER_PATCH", parts.next().unwrap()) + .env("CFG_VER_BUILD", "0") // just needed to build + .env("CFG_PACKAGE_VERS", builder.rust_package_vers()) + .env("CFG_PACKAGE_NAME", pkgname(builder, "rust")) + .env("CFG_BUILD", target.triple) + .env("CFG_CHANNEL", &builder.config.channel); + + if target.contains("windows-gnullvm") { + cmd.env("CFG_MINGW", "1").env("CFG_ABI", "LLVM"); + } else if target.contains("windows-gnu") { + cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU"); + } else { + cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC"); + } +} + +/// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking. +/// +/// Returns whether the files were actually copied. +fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool { + if let Some(config) = builder.config.target_config.get(&target) { + if config.llvm_config.is_some() && !builder.config.llvm_from_ci { + // If the LLVM was externally provided, then we don't currently copy + // artifacts into the sysroot. This is not necessarily the right + // choice (in particular, it will require the LLVM dylib to be in + // the linker's load path at runtime), but the common use case for + // external LLVMs is distribution provided LLVMs, and in that case + // they're usually in the standard search path (e.g., /usr/lib) and + // copying them here is going to cause problems as we may end up + // with the wrong files and isn't what distributions want. + // + // This behavior may be revisited in the future though. + // + // If the LLVM is coming from ourselves (just from CI) though, we + // still want to install it, as it otherwise won't be available. + return false; + } + } + + // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib + // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely + // clear why this is the case, though. llvm-config will emit the versioned + // paths and we don't want those in the sysroot (as we're expecting + // unversioned paths). + if target.contains("apple-darwin") && builder.llvm_link_shared() { + let src_libdir = builder.llvm_out(target).join("lib"); + let llvm_dylib_path = src_libdir.join("libLLVM.dylib"); + if llvm_dylib_path.exists() { + builder.install(&llvm_dylib_path, dst_libdir, 0o644); + } + !builder.config.dry_run + } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) { + let mut cmd = Command::new(llvm_config); + cmd.arg("--libfiles"); + builder.verbose(&format!("running {:?}", cmd)); + let files = output(&mut cmd); + let build_llvm_out = &builder.llvm_out(builder.config.build); + let target_llvm_out = &builder.llvm_out(target); + for file in files.trim_end().split(' ') { + // If we're not using a custom LLVM, make sure we package for the target. + let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) { + target_llvm_out.join(relative_path) + } else { + PathBuf::from(file) + }; + builder.install(&file, dst_libdir, 0o644); + } + !builder.config.dry_run + } else { + false + } +} + +/// Maybe add libLLVM.so to the target lib-dir for linking. +pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) { + let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib"); + // We do not need to copy LLVM files into the sysroot if it is not + // dynamically linked; it is already included into librustc_llvm + // statically. + if builder.llvm_link_shared() { + maybe_install_llvm(builder, target, &dst_libdir); + } +} + +/// Maybe add libLLVM.so to the runtime lib-dir for rustc itself. +pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) { + let dst_libdir = + sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target })); + // We do not need to copy LLVM files into the sysroot if it is not + // dynamically linked; it is already included into librustc_llvm + // statically. + if builder.llvm_link_shared() { + maybe_install_llvm(builder, target, &dst_libdir); + } +} + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct LlvmTools { + pub target: TargetSelection, +} + +impl Step for LlvmTools { + type Output = Option<GeneratedTarball>; + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(&run.builder, "llvm-tools"); + // FIXME: allow using the names of the tools themselves? + run.alias("llvm-tools").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(LlvmTools { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { + let target = self.target; + + /* run only if llvm-config isn't used */ + if let Some(config) = builder.config.target_config.get(&target) { + if let Some(ref _s) = config.llvm_config { + builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target)); + return None; + } + } + + builder.ensure(crate::native::Llvm { target }); + + let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple); + tarball.set_overlay(OverlayKind::LLVM); + tarball.is_preview(true); + + // Prepare the image directory + let src_bindir = builder.llvm_out(target).join("bin"); + let dst_bindir = format!("lib/rustlib/{}/bin", target.triple); + for tool in LLVM_TOOLS { + let exe = src_bindir.join(exe(tool, target)); + tarball.add_file(&exe, &dst_bindir, 0o755); + } + + // Copy libLLVM.so to the target lib dir as well, so the RPATH like + // `$ORIGIN/../lib` can find it. It may also be used as a dependency + // of `rustc-dev` to support the inherited `-lLLVM` when using the + // compiler libraries. + maybe_install_llvm_target(builder, target, tarball.image_dir()); + + Some(tarball.generate()) + } +} + +// Tarball intended for internal consumption to ease rustc/std development. +// +// Should not be considered stable by end users. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct RustDev { + pub target: TargetSelection, +} + +impl Step for RustDev { + type Output = Option<GeneratedTarball>; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("rust-dev") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustDev { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { + let target = self.target; + + /* run only if llvm-config isn't used */ + if let Some(config) = builder.config.target_config.get(&target) { + if let Some(ref _s) = config.llvm_config { + builder.info(&format!("Skipping RustDev ({}): external LLVM", target)); + return None; + } + } + + let mut tarball = Tarball::new(builder, "rust-dev", &target.triple); + tarball.set_overlay(OverlayKind::LLVM); + + builder.ensure(crate::native::Llvm { target }); + + let src_bindir = builder.llvm_out(target).join("bin"); + // If updating this list, you likely want to change + // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users + // will not pick up the extra file until LLVM gets bumped. + for bin in &[ + "llvm-config", + "llvm-ar", + "llvm-objdump", + "llvm-profdata", + "llvm-bcanalyzer", + "llvm-cov", + "llvm-dwp", + "llvm-nm", + "llvm-dwarfdump", + ] { + tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755); + } + + // We don't build LLD on some platforms, so only add it if it exists + let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target)); + if lld_path.exists() { + tarball.add_file(lld_path, "bin", 0o755); + } + + tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755); + + // Copy the include directory as well; needed mostly to build + // librustc_llvm properly (e.g., llvm-config.h is in here). But also + // just broadly useful to be able to link against the bundled LLVM. + tarball.add_dir(&builder.llvm_out(target).join("include"), "include"); + + // Copy libLLVM.so to the target lib dir as well, so the RPATH like + // `$ORIGIN/../lib` can find it. It may also be used as a dependency + // of `rustc-dev` to support the inherited `-lLLVM` when using the + // compiler libraries. + let dst_libdir = tarball.image_dir().join("lib"); + maybe_install_llvm(builder, target, &dst_libdir); + let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" }; + t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir); + + Some(tarball.generate()) + } +} + +/// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the +/// release process to avoid cloning the monorepo and building stuff. +/// +/// Should not be considered stable by end users. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct BuildManifest { + pub target: TargetSelection, +} + +impl Step for BuildManifest { + type Output = GeneratedTarball; + const DEFAULT: bool = false; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("build-manifest") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(BuildManifest { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + let build_manifest = builder.tool_exe(Tool::BuildManifest); + + let tarball = Tarball::new(builder, "build-manifest", &self.target.triple); + tarball.add_file(&build_manifest, "bin", 0o755); + tarball.generate() + } +} + +/// Tarball containing artifacts necessary to reproduce the build of rustc. +/// +/// Currently this is the PGO profile data. +/// +/// Should not be considered stable by end users. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct ReproducibleArtifacts { + pub target: TargetSelection, +} + +impl Step for ReproducibleArtifacts { + type Output = Option<GeneratedTarball>; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("reproducible-artifacts") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(ReproducibleArtifacts { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + let mut added_anything = false; + let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple); + if let Some(path) = builder.config.rust_profile_use.as_ref() { + tarball.add_file(path, ".", 0o644); + added_anything = true; + } + if let Some(path) = builder.config.llvm_profile_use.as_ref() { + tarball.add_file(path, ".", 0o644); + added_anything = true; + } + if added_anything { Some(tarball.generate()) } else { None } + } +} diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs new file mode 100644 index 000000000..2852442d0 --- /dev/null +++ b/src/bootstrap/doc.rs @@ -0,0 +1,932 @@ +//! Documentation generation for rustbuilder. +//! +//! This module implements generation for all bits and pieces of documentation +//! for the Rust project. This notably includes suites like the rust book, the +//! nomicon, rust by example, standalone documentation, etc. +//! +//! Everything here is basically just a shim around calling either `rustbook` or +//! `rustdoc`. + +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +use crate::builder::{Builder, Compiler, Kind, RunConfig, ShouldRun, Step}; +use crate::cache::{Interned, INTERNER}; +use crate::compile; +use crate::config::{Config, TargetSelection}; +use crate::tool::{self, prepare_tool_cargo, SourceType, Tool}; +use crate::util::{symlink_dir, t, up_to_date}; +use crate::Mode; + +macro_rules! submodule_helper { + ($path:expr, submodule) => { + $path + }; + ($path:expr, submodule = $submodule:literal) => { + $submodule + }; +} + +macro_rules! book { + ($($name:ident, $path:expr, $book_name:expr $(, submodule $(= $submodule:literal)? )? ;)+) => { + $( + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] + pub struct $name { + target: TargetSelection, + } + + impl Step for $name { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path($path).default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure($name { + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) { + $( + let path = Path::new(submodule_helper!( $path, submodule $( = $submodule )? )); + builder.update_submodule(&path); + )? + builder.ensure(RustbookSrc { + target: self.target, + name: INTERNER.intern_str($book_name), + src: INTERNER.intern_path(builder.src.join($path)), + }) + } + } + )+ + } +} + +// NOTE: When adding a book here, make sure to ALSO build the book by +// adding a build step in `src/bootstrap/builder.rs`! +// NOTE: Make sure to add the corresponding submodule when adding a new book. +// FIXME: Make checking for a submodule automatic somehow (maybe by having a list of all submodules +// and checking against it?). +book!( + CargoBook, "src/tools/cargo/src/doc", "cargo", submodule = "src/tools/cargo"; + ClippyBook, "src/tools/clippy/book", "clippy"; + EditionGuide, "src/doc/edition-guide", "edition-guide", submodule; + EmbeddedBook, "src/doc/embedded-book", "embedded-book", submodule; + Nomicon, "src/doc/nomicon", "nomicon", submodule; + Reference, "src/doc/reference", "reference", submodule; + RustByExample, "src/doc/rust-by-example", "rust-by-example", submodule; + RustdocBook, "src/doc/rustdoc", "rustdoc"; +); + +fn open(builder: &Builder<'_>, path: impl AsRef<Path>) { + if builder.config.dry_run || !builder.config.cmd.open() { + return; + } + + let path = path.as_ref(); + builder.info(&format!("Opening doc {}", path.display())); + if let Err(err) = opener::open(path) { + builder.info(&format!("{}\n", err)); + } +} + +// "library/std" -> ["library", "std"] +// +// Used for deciding whether a particular step is one requested by the user on +// the `x.py doc` command line, which determines whether `--open` will open that +// page. +pub(crate) fn components_simplified(path: &PathBuf) -> Vec<&str> { + path.iter().map(|component| component.to_str().unwrap_or("???")).collect() +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct UnstableBook { + target: TargetSelection, +} + +impl Step for UnstableBook { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/doc/unstable-book").default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(UnstableBook { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + builder.ensure(UnstableBookGen { target: self.target }); + builder.ensure(RustbookSrc { + target: self.target, + name: INTERNER.intern_str("unstable-book"), + src: INTERNER.intern_path(builder.md_doc_out(self.target).join("unstable-book")), + }) + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +struct RustbookSrc { + target: TargetSelection, + name: Interned<String>, + src: Interned<PathBuf>, +} + +impl Step for RustbookSrc { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Invoke `rustbook` for `target` for the doc book `name` from the `src` path. + /// + /// This will not actually generate any documentation if the documentation has + /// already been generated. + fn run(self, builder: &Builder<'_>) { + let target = self.target; + let name = self.name; + let src = self.src; + let out = builder.doc_out(target); + t!(fs::create_dir_all(&out)); + + let out = out.join(name); + let index = out.join("index.html"); + let rustbook = builder.tool_exe(Tool::Rustbook); + let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook); + if builder.config.dry_run || up_to_date(&src, &index) && up_to_date(&rustbook, &index) { + return; + } + builder.info(&format!("Rustbook ({}) - {}", target, name)); + let _ = fs::remove_dir_all(&out); + + builder.run(rustbook_cmd.arg("build").arg(&src).arg("-d").arg(out)); + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct TheBook { + compiler: Compiler, + target: TargetSelection, +} + +impl Step for TheBook { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/doc/book").default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(TheBook { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + /// Builds the book and associated stuff. + /// + /// We need to build: + /// + /// * Book + /// * Older edition redirects + /// * Version info and CSS + /// * Index page + /// * Redirect pages + fn run(self, builder: &Builder<'_>) { + let relative_path = Path::new("src").join("doc").join("book"); + builder.update_submodule(&relative_path); + + let compiler = self.compiler; + let target = self.target; + + // build book + builder.ensure(RustbookSrc { + target, + name: INTERNER.intern_str("book"), + src: INTERNER.intern_path(builder.src.join(&relative_path)), + }); + + // building older edition redirects + for edition in &["first-edition", "second-edition", "2018-edition"] { + builder.ensure(RustbookSrc { + target, + name: INTERNER.intern_string(format!("book/{}", edition)), + src: INTERNER.intern_path(builder.src.join(&relative_path).join(edition)), + }); + } + + // build the version info page and CSS + builder.ensure(Standalone { compiler, target }); + + // build the redirect pages + builder.info(&format!("Documenting book redirect pages ({})", target)); + for file in t!(fs::read_dir(builder.src.join(&relative_path).join("redirects"))) { + let file = t!(file); + let path = file.path(); + let path = path.to_str().unwrap(); + + invoke_rustdoc(builder, compiler, target, path); + } + + if builder.was_invoked_explicitly::<Self>(Kind::Doc) { + let out = builder.doc_out(target); + let index = out.join("book").join("index.html"); + open(builder, &index); + } + } +} + +fn invoke_rustdoc( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, + markdown: &str, +) { + let out = builder.doc_out(target); + + let path = builder.src.join("src/doc").join(markdown); + + let header = builder.src.join("src/doc/redirect.inc"); + let footer = builder.src.join("src/doc/footer.inc"); + let version_info = out.join("version_info.html"); + + let mut cmd = builder.rustdoc_cmd(compiler); + + let out = out.join("book"); + + cmd.arg("--html-after-content") + .arg(&footer) + .arg("--html-before-content") + .arg(&version_info) + .arg("--html-in-header") + .arg(&header) + .arg("--markdown-no-toc") + .arg("--markdown-playground-url") + .arg("https://play.rust-lang.org/") + .arg("-o") + .arg(&out) + .arg(&path) + .arg("--markdown-css") + .arg("../rust.css"); + + if !builder.config.docs_minification { + cmd.arg("-Z").arg("unstable-options").arg("--disable-minification"); + } + + builder.run(&mut cmd); +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Standalone { + compiler: Compiler, + target: TargetSelection, +} + +impl Step for Standalone { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/doc").default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Standalone { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + /// Generates all standalone documentation as compiled by the rustdoc in `stage` + /// for the `target` into `out`. + /// + /// This will list all of `src/doc` looking for markdown files and appropriately + /// perform transformations like substituting `VERSION`, `SHORT_HASH`, and + /// `STAMP` along with providing the various header/footer HTML we've customized. + /// + /// In the end, this is just a glorified wrapper around rustdoc! + fn run(self, builder: &Builder<'_>) { + let target = self.target; + let compiler = self.compiler; + builder.info(&format!("Documenting standalone ({})", target)); + let out = builder.doc_out(target); + t!(fs::create_dir_all(&out)); + + let favicon = builder.src.join("src/doc/favicon.inc"); + let footer = builder.src.join("src/doc/footer.inc"); + let full_toc = builder.src.join("src/doc/full-toc.inc"); + t!(fs::copy(builder.src.join("src/doc/rust.css"), out.join("rust.css"))); + + let version_input = builder.src.join("src/doc/version_info.html.template"); + let version_info = out.join("version_info.html"); + + if !builder.config.dry_run && !up_to_date(&version_input, &version_info) { + let info = t!(fs::read_to_string(&version_input)) + .replace("VERSION", &builder.rust_release()) + .replace("SHORT_HASH", builder.rust_info.sha_short().unwrap_or("")) + .replace("STAMP", builder.rust_info.sha().unwrap_or("")); + t!(fs::write(&version_info, &info)); + } + + for file in t!(fs::read_dir(builder.src.join("src/doc"))) { + let file = t!(file); + let path = file.path(); + let filename = path.file_name().unwrap().to_str().unwrap(); + if !filename.ends_with(".md") || filename == "README.md" { + continue; + } + + let html = out.join(filename).with_extension("html"); + let rustdoc = builder.rustdoc(compiler); + if up_to_date(&path, &html) + && up_to_date(&footer, &html) + && up_to_date(&favicon, &html) + && up_to_date(&full_toc, &html) + && (builder.config.dry_run || up_to_date(&version_info, &html)) + && (builder.config.dry_run || up_to_date(&rustdoc, &html)) + { + continue; + } + + let mut cmd = builder.rustdoc_cmd(compiler); + // Needed for --index-page flag + cmd.arg("-Z").arg("unstable-options"); + + cmd.arg("--html-after-content") + .arg(&footer) + .arg("--html-before-content") + .arg(&version_info) + .arg("--html-in-header") + .arg(&favicon) + .arg("--markdown-no-toc") + .arg("--index-page") + .arg(&builder.src.join("src/doc/index.md")) + .arg("--markdown-playground-url") + .arg("https://play.rust-lang.org/") + .arg("-o") + .arg(&out) + .arg(&path); + + if !builder.config.docs_minification { + cmd.arg("--disable-minification"); + } + + if filename == "not_found.md" { + cmd.arg("--markdown-css") + .arg(format!("https://doc.rust-lang.org/rustdoc{}.css", &builder.version)) + .arg("--markdown-css") + .arg("https://doc.rust-lang.org/rust.css"); + } else { + cmd.arg("--markdown-css") + .arg(format!("rustdoc{}.css", &builder.version)) + .arg("--markdown-css") + .arg("rust.css"); + } + builder.run(&mut cmd); + } + + // We open doc/index.html as the default if invoked as `x.py doc --open` + // with no particular explicit doc requested (e.g. library/core). + if builder.paths.is_empty() || builder.was_invoked_explicitly::<Self>(Kind::Doc) { + let index = out.join("index.html"); + open(builder, &index); + } + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Std { + pub stage: u32, + pub target: TargetSelection, +} + +impl Step for Std { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.all_krates("test").path("library").default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Std { stage: run.builder.top_stage, target: run.target }); + } + + /// Compile all standard library documentation. + /// + /// This will generate all documentation for the standard library and its + /// dependencies. This is largely just a wrapper around `cargo doc`. + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let target = self.target; + builder.info(&format!("Documenting stage{} std ({})", stage, target)); + if builder.no_std(target) == Some(true) { + panic!( + "building std documentation for no_std target {target} is not supported\n\ + Set `docs = false` in the config to disable documentation." + ); + } + let out = builder.doc_out(target); + t!(fs::create_dir_all(&out)); + let compiler = builder.compiler(stage, builder.config.build); + + let out_dir = builder.stage_out(compiler, Mode::Std).join(target.triple).join("doc"); + + t!(fs::copy(builder.src.join("src/doc/rust.css"), out.join("rust.css"))); + + let run_cargo_rustdoc_for = |package: &str| { + let mut cargo = + builder.cargo(compiler, Mode::Std, SourceType::InTree, target, "rustdoc"); + compile::std_cargo(builder, target, compiler.stage, &mut cargo); + + cargo + .arg("-p") + .arg(package) + .arg("-Zskip-rustdoc-fingerprint") + .arg("--") + .arg("--markdown-css") + .arg("rust.css") + .arg("--markdown-no-toc") + .arg("-Z") + .arg("unstable-options") + .arg("--resource-suffix") + .arg(&builder.version) + .arg("--index-page") + .arg(&builder.src.join("src/doc/index.md")); + + if !builder.config.docs_minification { + cargo.arg("--disable-minification"); + } + + builder.run(&mut cargo.into()); + }; + + let paths = builder + .paths + .iter() + .map(components_simplified) + .filter_map(|path| { + if path.len() >= 2 && path.get(0) == Some(&"library") { + // single crate + Some(path[1].to_owned()) + } else if !path.is_empty() { + // ?? + Some(path[0].to_owned()) + } else { + // all library crates + None + } + }) + .collect::<Vec<_>>(); + + // Only build the following crates. While we could just iterate over the + // folder structure, that would also build internal crates that we do + // not want to show in documentation. These crates will later be visited + // by the rustc step, so internal documentation will show them. + // + // Note that the order here is important! The crates need to be + // processed starting from the leaves, otherwise rustdoc will not + // create correct links between crates because rustdoc depends on the + // existence of the output directories to know if it should be a local + // or remote link. + let krates = ["core", "alloc", "std", "proc_macro", "test"]; + for krate in &krates { + run_cargo_rustdoc_for(krate); + if paths.iter().any(|p| p == krate) { + // No need to document more of the libraries if we have the one we want. + break; + } + } + builder.cp_r(&out_dir, &out); + + // Look for library/std, library/core etc in the `x.py doc` arguments and + // open the corresponding rendered docs. + for requested_crate in paths { + if krates.iter().any(|k| *k == requested_crate.as_str()) { + let index = out.join(requested_crate).join("index.html"); + open(builder, &index); + } + } + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Rustc { + pub stage: u32, + pub target: TargetSelection, +} + +impl Step for Rustc { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.crate_or_deps("rustc-main") + .path("compiler") + .default_condition(builder.config.compiler_docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Rustc { stage: run.builder.top_stage, target: run.target }); + } + + /// Generates compiler documentation. + /// + /// This will generate all documentation for compiler and dependencies. + /// Compiler documentation is distributed separately, so we make sure + /// we do not merge it with the other documentation from std, test and + /// proc_macros. This is largely just a wrapper around `cargo doc`. + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let target = self.target; + + let paths = builder + .paths + .iter() + .filter(|path| { + let components = components_simplified(path); + components.len() >= 2 && components[0] == "compiler" + }) + .collect::<Vec<_>>(); + + // This is the intended out directory for compiler documentation. + let out = builder.compiler_doc_out(target); + t!(fs::create_dir_all(&out)); + + // Build the standard library, so that proc-macros can use it. + // (Normally, only the metadata would be necessary, but proc-macros are special since they run at compile-time.) + let compiler = builder.compiler(stage, builder.config.build); + builder.ensure(compile::Std::new(compiler, builder.config.build)); + + builder.info(&format!("Documenting stage{} compiler ({})", stage, target)); + + // This uses a shared directory so that librustdoc documentation gets + // correctly built and merged with the rustc documentation. This is + // needed because rustdoc is built in a different directory from + // rustc. rustdoc needs to be able to see everything, for example when + // merging the search index, or generating local (relative) links. + let out_dir = builder.stage_out(compiler, Mode::Rustc).join(target.triple).join("doc"); + t!(symlink_dir_force(&builder.config, &out, &out_dir)); + // Cargo puts proc macros in `target/doc` even if you pass `--target` + // explicitly (https://github.com/rust-lang/cargo/issues/7677). + let proc_macro_out_dir = builder.stage_out(compiler, Mode::Rustc).join("doc"); + t!(symlink_dir_force(&builder.config, &out, &proc_macro_out_dir)); + + // Build cargo command. + let mut cargo = builder.cargo(compiler, Mode::Rustc, SourceType::InTree, target, "doc"); + cargo.rustdocflag("--document-private-items"); + // Since we always pass --document-private-items, there's no need to warn about linking to private items. + cargo.rustdocflag("-Arustdoc::private-intra-doc-links"); + cargo.rustdocflag("--enable-index-page"); + cargo.rustdocflag("-Zunstable-options"); + cargo.rustdocflag("-Znormalize-docs"); + cargo.rustdocflag("--show-type-layout"); + cargo.rustdocflag("--generate-link-to-definition"); + compile::rustc_cargo(builder, &mut cargo, target); + cargo.arg("-Zunstable-options"); + cargo.arg("-Zskip-rustdoc-fingerprint"); + + // Only include compiler crates, no dependencies of those, such as `libc`. + // Do link to dependencies on `docs.rs` however using `rustdoc-map`. + cargo.arg("--no-deps"); + cargo.arg("-Zrustdoc-map"); + + // FIXME: `-Zrustdoc-map` does not yet correctly work for transitive dependencies, + // once this is no longer an issue the special case for `ena` can be removed. + cargo.rustdocflag("--extern-html-root-url"); + cargo.rustdocflag("ena=https://docs.rs/ena/latest/"); + + let root_crates = if paths.is_empty() { + vec![ + INTERNER.intern_str("rustc_driver"), + INTERNER.intern_str("rustc_codegen_llvm"), + INTERNER.intern_str("rustc_codegen_ssa"), + ] + } else { + paths.into_iter().map(|p| builder.crate_paths[p]).collect() + }; + // Find dependencies for top level crates. + let compiler_crates = root_crates.iter().flat_map(|krate| { + builder.in_tree_crates(krate, Some(target)).into_iter().map(|krate| krate.name) + }); + + let mut to_open = None; + for krate in compiler_crates { + // Create all crate output directories first to make sure rustdoc uses + // relative links. + // FIXME: Cargo should probably do this itself. + t!(fs::create_dir_all(out_dir.join(krate))); + cargo.arg("-p").arg(krate); + if to_open.is_none() { + to_open = Some(krate); + } + } + + builder.run(&mut cargo.into()); + // Let's open the first crate documentation page: + if let Some(krate) = to_open { + let index = out.join(krate).join("index.html"); + open(builder, &index); + } + } +} + +macro_rules! tool_doc { + ($tool: ident, $should_run: literal, $path: literal, [$($krate: literal),+ $(,)?], in_tree = $in_tree:expr $(,)?) => { + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] + pub struct $tool { + target: TargetSelection, + } + + impl Step for $tool { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.crate_or_deps($should_run).default_condition(builder.config.compiler_docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure($tool { target: run.target }); + } + + /// Generates compiler documentation. + /// + /// This will generate all documentation for compiler and dependencies. + /// Compiler documentation is distributed separately, so we make sure + /// we do not merge it with the other documentation from std, test and + /// proc_macros. This is largely just a wrapper around `cargo doc`. + fn run(self, builder: &Builder<'_>) { + let stage = builder.top_stage; + let target = self.target; + + // This is the intended out directory for compiler documentation. + let out = builder.compiler_doc_out(target); + t!(fs::create_dir_all(&out)); + + // Build rustc docs so that we generate relative links. + builder.ensure(Rustc { stage, target }); + // Rustdoc needs the rustc sysroot available to build. + // FIXME: is there a way to only ensure `check::Rustc` here? Last time I tried it failed + // with strange errors, but only on a full bors test ... + let compiler = builder.compiler(stage, builder.config.build); + builder.ensure(compile::Rustc::new(compiler, target)); + + builder.info( + &format!( + "Documenting stage{} {} ({})", + stage, + stringify!($tool).to_lowercase(), + target, + ), + ); + + // Symlink compiler docs to the output directory of rustdoc documentation. + let out_dir = builder.stage_out(compiler, Mode::ToolRustc).join(target.triple).join("doc"); + t!(fs::create_dir_all(&out_dir)); + t!(symlink_dir_force(&builder.config, &out, &out_dir)); + + let source_type = if $in_tree == true { + SourceType::InTree + } else { + SourceType::Submodule + }; + + // Build cargo command. + let mut cargo = prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + target, + "doc", + $path, + source_type, + &[], + ); + + cargo.arg("-Zskip-rustdoc-fingerprint"); + // Only include compiler crates, no dependencies of those, such as `libc`. + cargo.arg("--no-deps"); + $( + cargo.arg("-p").arg($krate); + )+ + + cargo.rustdocflag("--document-private-items"); + cargo.rustdocflag("--enable-index-page"); + cargo.rustdocflag("--show-type-layout"); + cargo.rustdocflag("--generate-link-to-definition"); + cargo.rustdocflag("-Zunstable-options"); + if $in_tree == true { + builder.run(&mut cargo.into()); + } else { + // Allow out-of-tree docs to fail (since the tool might be in a broken state). + if !builder.try_run(&mut cargo.into()) { + builder.info(&format!( + "WARNING: tool {} failed to document; ignoring failure because it is an out-of-tree tool", + stringify!($tool).to_lowercase(), + )); + } + } + } + } + } +} + +tool_doc!( + Rustdoc, + "rustdoc-tool", + "src/tools/rustdoc", + ["rustdoc", "rustdoc-json-types"], + in_tree = true +); +tool_doc!( + Rustfmt, + "rustfmt-nightly", + "src/tools/rustfmt", + ["rustfmt-nightly", "rustfmt-config_proc_macro"], + in_tree = true +); +tool_doc!(Clippy, "clippy", "src/tools/clippy", ["clippy_utils"], in_tree = true); +tool_doc!(Miri, "miri", "src/tools/miri", ["miri"], in_tree = false); + +#[derive(Ord, PartialOrd, Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct ErrorIndex { + pub target: TargetSelection, +} + +impl Step for ErrorIndex { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/tools/error_index_generator").default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + let target = run.target; + run.builder.ensure(ErrorIndex { target }); + } + + /// Generates the HTML rendered error-index by running the + /// `error_index_generator` tool. + fn run(self, builder: &Builder<'_>) { + builder.info(&format!("Documenting error index ({})", self.target)); + let out = builder.doc_out(self.target); + t!(fs::create_dir_all(&out)); + let mut index = tool::ErrorIndex::command(builder); + index.arg("html"); + index.arg(out.join("error-index.html")); + index.arg(&builder.version); + + builder.run(&mut index); + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct UnstableBookGen { + target: TargetSelection, +} + +impl Step for UnstableBookGen { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/tools/unstable-book-gen").default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(UnstableBookGen { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let target = self.target; + + builder.info(&format!("Generating unstable book md files ({})", target)); + let out = builder.md_doc_out(target).join("unstable-book"); + builder.create_dir(&out); + builder.remove_dir(&out); + let mut cmd = builder.tool_cmd(Tool::UnstableBookGen); + cmd.arg(builder.src.join("library")); + cmd.arg(builder.src.join("compiler")); + cmd.arg(builder.src.join("src")); + cmd.arg(out); + + builder.run(&mut cmd); + } +} + +fn symlink_dir_force(config: &Config, src: &Path, dst: &Path) -> io::Result<()> { + if config.dry_run { + return Ok(()); + } + if let Ok(m) = fs::symlink_metadata(dst) { + if m.file_type().is_dir() { + fs::remove_dir_all(dst)?; + } else { + // handle directory junctions on windows by falling back to + // `remove_dir`. + fs::remove_file(dst).or_else(|_| fs::remove_dir(dst))?; + } + } + + symlink_dir(config, src, dst) +} + +#[derive(Ord, PartialOrd, Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustcBook { + pub compiler: Compiler, + pub target: TargetSelection, + pub validate: bool, +} + +impl Step for RustcBook { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/doc/rustc").default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustcBook { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + validate: false, + }); + } + + /// Builds the rustc book. + /// + /// The lints are auto-generated by a tool, and then merged into the book + /// in the "md-doc" directory in the build output directory. Then + /// "rustbook" is used to convert it to HTML. + fn run(self, builder: &Builder<'_>) { + let out_base = builder.md_doc_out(self.target).join("rustc"); + t!(fs::create_dir_all(&out_base)); + let out_listing = out_base.join("src/lints"); + builder.cp_r(&builder.src.join("src/doc/rustc"), &out_base); + builder.info(&format!("Generating lint docs ({})", self.target)); + + let rustc = builder.rustc(self.compiler); + // The tool runs `rustc` for extracting output examples, so it needs a + // functional sysroot. + builder.ensure(compile::Std::new(self.compiler, self.target)); + let mut cmd = builder.tool_cmd(Tool::LintDocs); + cmd.arg("--src"); + cmd.arg(builder.src.join("compiler")); + cmd.arg("--out"); + cmd.arg(&out_listing); + cmd.arg("--rustc"); + cmd.arg(&rustc); + cmd.arg("--rustc-target").arg(&self.target.rustc_target_arg()); + if builder.config.verbose() { + cmd.arg("--verbose"); + } + if self.validate { + cmd.arg("--validate"); + } + if !builder.unstable_features() { + // We need to validate nightly features, even on the stable channel. + cmd.env("RUSTC_BOOTSTRAP", "1"); + } + // If the lib directories are in an unusual location (changed in + // config.toml), then this needs to explicitly update the dylib search + // path. + builder.add_rustc_lib_path(self.compiler, &mut cmd); + builder.run(&mut cmd); + // Run rustbook/mdbook to generate the HTML pages. + builder.ensure(RustbookSrc { + target: self.target, + name: INTERNER.intern_str("rustc"), + src: INTERNER.intern_path(out_base), + }); + if builder.was_invoked_explicitly::<Self>(Kind::Doc) { + let out = builder.doc_out(self.target); + let index = out.join("rustc").join("index.html"); + open(builder, &index); + } + } +} diff --git a/src/bootstrap/download-ci-llvm-stamp b/src/bootstrap/download-ci-llvm-stamp new file mode 100644 index 000000000..19504a51a --- /dev/null +++ b/src/bootstrap/download-ci-llvm-stamp @@ -0,0 +1,4 @@ +Change this file to make users of the `download-ci-llvm` configuration download +a new version of LLVM from CI, even if the LLVM submodule hasn’t changed. + +Last change is for: https://github.com/rust-lang/rust/pull/96867 diff --git a/src/bootstrap/dylib_util.rs b/src/bootstrap/dylib_util.rs new file mode 100644 index 000000000..6d75272c5 --- /dev/null +++ b/src/bootstrap/dylib_util.rs @@ -0,0 +1,28 @@ +// Various utilities for working with dylib paths. +// +// This file is meant to be included directly to avoid a dependency on the bootstrap library from +// the rustc and rustdoc wrappers. This improves compilation time by reducing the linking time. + +/// Returns the environment variable which the dynamic library lookup path +/// resides in for this platform. +pub fn dylib_path_var() -> &'static str { + if cfg!(target_os = "windows") { + "PATH" + } else if cfg!(target_os = "macos") { + "DYLD_LIBRARY_PATH" + } else if cfg!(target_os = "haiku") { + "LIBRARY_PATH" + } else { + "LD_LIBRARY_PATH" + } +} + +/// Parses the `dylib_path_var()` environment variable, returning a list of +/// paths that are members of this lookup path. +pub fn dylib_path() -> Vec<PathBuf> { + let var = match env::var_os(dylib_path_var()) { + Some(v) => v, + None => return vec![], + }; + env::split_paths(&var).collect() +} diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs new file mode 100644 index 000000000..80b3bcce8 --- /dev/null +++ b/src/bootstrap/flags.rs @@ -0,0 +1,817 @@ +//! Command-line interface of the rustbuild build system. +//! +//! This module implements the command-line parsing of the build system which +//! has various flags to configure how it's run. + +use std::path::PathBuf; + +use getopts::Options; + +use crate::builder::{Builder, Kind}; +use crate::config::{Config, TargetSelection}; +use crate::setup::Profile; +use crate::util::t; +use crate::{Build, DocTests}; + +#[derive(Copy, Clone)] +pub enum Color { + Always, + Never, + Auto, +} + +impl Default for Color { + fn default() -> Self { + Self::Auto + } +} + +impl std::str::FromStr for Color { + type Err = (); + + fn from_str(s: &str) -> Result<Self, Self::Err> { + match s.to_lowercase().as_str() { + "always" => Ok(Self::Always), + "never" => Ok(Self::Never), + "auto" => Ok(Self::Auto), + _ => Err(()), + } + } +} + +/// Deserialized version of all flags for this compile. +pub struct Flags { + pub verbose: usize, // number of -v args; each extra -v after the first is passed to Cargo + pub on_fail: Option<String>, + pub stage: Option<u32>, + pub keep_stage: Vec<u32>, + pub keep_stage_std: Vec<u32>, + + pub host: Option<Vec<TargetSelection>>, + pub target: Option<Vec<TargetSelection>>, + pub config: Option<PathBuf>, + pub build_dir: Option<PathBuf>, + pub jobs: Option<u32>, + pub cmd: Subcommand, + pub incremental: bool, + pub exclude: Vec<PathBuf>, + pub include_default_paths: bool, + pub rustc_error_format: Option<String>, + pub json_output: bool, + pub dry_run: bool, + pub color: Color, + + // This overrides the deny-warnings configuration option, + // which passes -Dwarnings to the compiler invocations. + // + // true => deny, false => warn + pub deny_warnings: Option<bool>, + + pub llvm_skip_rebuild: Option<bool>, + + pub rust_profile_use: Option<String>, + pub rust_profile_generate: Option<String>, + + pub llvm_profile_use: Option<String>, + // LLVM doesn't support a custom location for generating profile + // information. + // + // llvm_out/build/profiles/ is the location this writes to. + pub llvm_profile_generate: bool, +} + +#[cfg_attr(test, derive(Clone))] +pub enum Subcommand { + Build { + paths: Vec<PathBuf>, + }, + Check { + paths: Vec<PathBuf>, + }, + Clippy { + fix: bool, + paths: Vec<PathBuf>, + clippy_lint_allow: Vec<String>, + clippy_lint_deny: Vec<String>, + clippy_lint_warn: Vec<String>, + clippy_lint_forbid: Vec<String>, + }, + Fix { + paths: Vec<PathBuf>, + }, + Format { + paths: Vec<PathBuf>, + check: bool, + }, + Doc { + paths: Vec<PathBuf>, + open: bool, + }, + Test { + paths: Vec<PathBuf>, + /// Whether to automatically update stderr/stdout files + bless: bool, + force_rerun: bool, + compare_mode: Option<String>, + pass: Option<String>, + run: Option<String>, + skip: Vec<String>, + test_args: Vec<String>, + rustc_args: Vec<String>, + fail_fast: bool, + doc_tests: DocTests, + rustfix_coverage: bool, + }, + Bench { + paths: Vec<PathBuf>, + test_args: Vec<String>, + }, + Clean { + all: bool, + }, + Dist { + paths: Vec<PathBuf>, + }, + Install { + paths: Vec<PathBuf>, + }, + Run { + paths: Vec<PathBuf>, + }, + Setup { + profile: Profile, + }, +} + +impl Default for Subcommand { + fn default() -> Subcommand { + Subcommand::Build { paths: vec![PathBuf::from("nowhere")] } + } +} + +impl Flags { + pub fn parse(args: &[String]) -> Flags { + let mut subcommand_help = String::from( + "\ +Usage: x.py <subcommand> [options] [<paths>...] + +Subcommands: + build, b Compile either the compiler or libraries + check, c Compile either the compiler or libraries, using cargo check + clippy Run clippy (uses rustup/cargo-installed clippy binary) + fix Run cargo fix + fmt Run rustfmt + test, t Build and run some test suites + bench Build and run some benchmarks + doc, d Build documentation + clean Clean out build directories + dist Build distribution artifacts + install Install distribution artifacts + run, r Run tools contained in this repository + setup Create a config.toml (making it easier to use `x.py` itself) + +To learn more about a subcommand, run `./x.py <subcommand> -h`", + ); + + let mut opts = Options::new(); + // Options common to all subcommands + opts.optflagmulti("v", "verbose", "use verbose output (-vv for very verbose)"); + opts.optflag("i", "incremental", "use incremental compilation"); + opts.optopt("", "config", "TOML configuration file for build", "FILE"); + opts.optopt( + "", + "build-dir", + "Build directory, overrides `build.build-dir` in `config.toml`", + "DIR", + ); + opts.optopt("", "build", "build target of the stage0 compiler", "BUILD"); + opts.optmulti("", "host", "host targets to build", "HOST"); + opts.optmulti("", "target", "target targets to build", "TARGET"); + opts.optmulti("", "exclude", "build paths to exclude", "PATH"); + opts.optflag( + "", + "include-default-paths", + "include default paths in addition to the provided ones", + ); + opts.optopt("", "on-fail", "command to run on failure", "CMD"); + opts.optflag("", "dry-run", "dry run; don't build anything"); + opts.optopt( + "", + "stage", + "stage to build (indicates compiler to use/test, e.g., stage 0 uses the \ + bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.)", + "N", + ); + opts.optmulti( + "", + "keep-stage", + "stage(s) to keep without recompiling \ + (pass multiple times to keep e.g., both stages 0 and 1)", + "N", + ); + opts.optmulti( + "", + "keep-stage-std", + "stage(s) of the standard library to keep without recompiling \ + (pass multiple times to keep e.g., both stages 0 and 1)", + "N", + ); + opts.optopt("", "src", "path to the root of the rust checkout", "DIR"); + let j_msg = format!( + "number of jobs to run in parallel; \ + defaults to {} (this host's logical CPU count)", + num_cpus::get() + ); + opts.optopt("j", "jobs", &j_msg, "JOBS"); + opts.optflag("h", "help", "print this help message"); + opts.optopt( + "", + "warnings", + "if value is deny, will deny warnings, otherwise use default", + "VALUE", + ); + opts.optopt("", "error-format", "rustc error format", "FORMAT"); + opts.optflag("", "json-output", "use message-format=json"); + opts.optopt("", "color", "whether to use color in cargo and rustc output", "STYLE"); + opts.optopt( + "", + "llvm-skip-rebuild", + "whether rebuilding llvm should be skipped \ + a VALUE of TRUE indicates that llvm will not be rebuilt \ + VALUE overrides the skip-rebuild option in config.toml.", + "VALUE", + ); + opts.optopt( + "", + "rust-profile-generate", + "generate PGO profile with rustc build", + "PROFILE", + ); + opts.optopt("", "rust-profile-use", "use PGO profile for rustc build", "PROFILE"); + opts.optflag("", "llvm-profile-generate", "generate PGO profile with llvm built for rustc"); + opts.optopt("", "llvm-profile-use", "use PGO profile for llvm build", "PROFILE"); + opts.optmulti("A", "", "allow certain clippy lints", "OPT"); + opts.optmulti("D", "", "deny certain clippy lints", "OPT"); + opts.optmulti("W", "", "warn about certain clippy lints", "OPT"); + opts.optmulti("F", "", "forbid certain clippy lints", "OPT"); + + // We can't use getopt to parse the options until we have completed specifying which + // options are valid, but under the current implementation, some options are conditional on + // the subcommand. Therefore we must manually identify the subcommand first, so that we can + // complete the definition of the options. Then we can use the getopt::Matches object from + // there on out. + let subcommand = match args.iter().find_map(|s| Kind::parse(&s)) { + Some(s) => s, + None => { + // No or an invalid subcommand -- show the general usage and subcommand help + // An exit code will be 0 when no subcommand is given, and 1 in case of an invalid + // subcommand. + println!("{}\n", subcommand_help); + let exit_code = if args.is_empty() { 0 } else { 1 }; + crate::detail_exit(exit_code); + } + }; + + // Some subcommands get extra options + match subcommand { + Kind::Test => { + opts.optflag("", "no-fail-fast", "Run all tests regardless of failure"); + opts.optmulti("", "skip", "skips tests matching SUBSTRING, if supported by test tool. May be passed multiple times", "SUBSTRING"); + opts.optmulti( + "", + "test-args", + "extra arguments to be passed for the test tool being used \ + (e.g. libtest, compiletest or rustdoc)", + "ARGS", + ); + opts.optmulti( + "", + "rustc-args", + "extra options to pass the compiler when running tests", + "ARGS", + ); + opts.optflag("", "no-doc", "do not run doc tests"); + opts.optflag("", "doc", "only run doc tests"); + opts.optflag("", "bless", "update all stderr/stdout files of failing ui tests"); + opts.optflag("", "force-rerun", "rerun tests even if the inputs are unchanged"); + opts.optopt( + "", + "compare-mode", + "mode describing what file the actual ui output will be compared to", + "COMPARE MODE", + ); + opts.optopt( + "", + "pass", + "force {check,build,run}-pass tests to this mode.", + "check | build | run", + ); + opts.optopt("", "run", "whether to execute run-* tests", "auto | always | never"); + opts.optflag( + "", + "rustfix-coverage", + "enable this to generate a Rustfix coverage file, which is saved in \ + `/<build_base>/rustfix_missing_coverage.txt`", + ); + } + Kind::Check => { + opts.optflag("", "all-targets", "Check all targets"); + } + Kind::Bench => { + opts.optmulti("", "test-args", "extra arguments", "ARGS"); + } + Kind::Clippy => { + opts.optflag("", "fix", "automatically apply lint suggestions"); + } + Kind::Doc => { + opts.optflag("", "open", "open the docs in a browser"); + } + Kind::Clean => { + opts.optflag("", "all", "clean all build artifacts"); + } + Kind::Format => { + opts.optflag("", "check", "check formatting instead of applying."); + } + _ => {} + }; + + // fn usage() + let usage = |exit_code: i32, opts: &Options, verbose: bool, subcommand_help: &str| -> ! { + let config = Config::parse(&["build".to_string()]); + let build = Build::new(config); + let paths = Builder::get_help(&build, subcommand); + + println!("{}", opts.usage(subcommand_help)); + if let Some(s) = paths { + if verbose { + println!("{}", s); + } else { + println!( + "Run `./x.py {} -h -v` to see a list of available paths.", + subcommand.as_str() + ); + } + } else if verbose { + panic!("No paths available for subcommand `{}`", subcommand.as_str()); + } + crate::detail_exit(exit_code); + }; + + // Done specifying what options are possible, so do the getopts parsing + let matches = opts.parse(args).unwrap_or_else(|e| { + // Invalid argument/option format + println!("\n{}\n", e); + usage(1, &opts, false, &subcommand_help); + }); + + // Extra sanity check to make sure we didn't hit this crazy corner case: + // + // ./x.py --frobulate clean build + // ^-- option ^ ^- actual subcommand + // \_ arg to option could be mistaken as subcommand + let mut pass_sanity_check = true; + match matches.free.get(0).and_then(|s| Kind::parse(&s)) { + Some(check_subcommand) => { + if check_subcommand != subcommand { + pass_sanity_check = false; + } + } + None => { + pass_sanity_check = false; + } + } + if !pass_sanity_check { + eprintln!("{}\n", subcommand_help); + eprintln!( + "Sorry, I couldn't figure out which subcommand you were trying to specify.\n\ + You may need to move some options to after the subcommand.\n" + ); + crate::detail_exit(1); + } + // Extra help text for some commands + match subcommand { + Kind::Build => { + subcommand_help.push_str( + "\n +Arguments: + This subcommand accepts a number of paths to directories to the crates + and/or artifacts to compile. For example, for a quick build of a usable + compiler: + + ./x.py build --stage 1 library/std + + This will build a compiler and standard library from the local source code. + Once this is done, build/$ARCH/stage1 contains a usable compiler. + + If no arguments are passed then the default artifacts for that stage are + compiled. For example: + + ./x.py build --stage 0 + ./x.py build ", + ); + } + Kind::Check => { + subcommand_help.push_str( + "\n +Arguments: + This subcommand accepts a number of paths to directories to the crates + and/or artifacts to compile. For example: + + ./x.py check library/std + + If no arguments are passed then many artifacts are checked.", + ); + } + Kind::Clippy => { + subcommand_help.push_str( + "\n +Arguments: + This subcommand accepts a number of paths to directories to the crates + and/or artifacts to run clippy against. For example: + + ./x.py clippy library/core + ./x.py clippy library/core library/proc_macro", + ); + } + Kind::Fix => { + subcommand_help.push_str( + "\n +Arguments: + This subcommand accepts a number of paths to directories to the crates + and/or artifacts to run `cargo fix` against. For example: + + ./x.py fix library/core + ./x.py fix library/core library/proc_macro", + ); + } + Kind::Format => { + subcommand_help.push_str( + "\n +Arguments: + This subcommand optionally accepts a `--check` flag which succeeds if formatting is correct and + fails if it is not. For example: + + ./x.py fmt + ./x.py fmt --check", + ); + } + Kind::Test => { + subcommand_help.push_str( + "\n +Arguments: + This subcommand accepts a number of paths to test directories that + should be compiled and run. For example: + + ./x.py test src/test/ui + ./x.py test library/std --test-args hash_map + ./x.py test library/std --stage 0 --no-doc + ./x.py test src/test/ui --bless + ./x.py test src/test/ui --compare-mode chalk + + Note that `test src/test/* --stage N` does NOT depend on `build compiler/rustc --stage N`; + just like `build library/std --stage N` it tests the compiler produced by the previous + stage. + + Execute tool tests with a tool name argument: + + ./x.py test tidy + + If no arguments are passed then the complete artifacts for that stage are + compiled and tested. + + ./x.py test + ./x.py test --stage 1", + ); + } + Kind::Doc => { + subcommand_help.push_str( + "\n +Arguments: + This subcommand accepts a number of paths to directories of documentation + to build. For example: + + ./x.py doc src/doc/book + ./x.py doc src/doc/nomicon + ./x.py doc src/doc/book library/std + ./x.py doc library/std --open + + If no arguments are passed then everything is documented: + + ./x.py doc + ./x.py doc --stage 1", + ); + } + Kind::Run => { + subcommand_help.push_str( + "\n +Arguments: + This subcommand accepts a number of paths to tools to build and run. For + example: + + ./x.py run src/tools/expand-yaml-anchors + + At least a tool needs to be called.", + ); + } + Kind::Setup => { + subcommand_help.push_str(&format!( + "\n +x.py setup creates a `config.toml` which changes the defaults for x.py itself. + +Arguments: + This subcommand accepts a 'profile' to use for builds. For example: + + ./x.py setup library + + The profile is optional and you will be prompted interactively if it is not given. + The following profiles are available: + +{}", + Profile::all_for_help(" ").trim_end() + )); + } + Kind::Bench | Kind::Clean | Kind::Dist | Kind::Install => {} + }; + // Get any optional paths which occur after the subcommand + let mut paths = matches.free[1..].iter().map(|p| p.into()).collect::<Vec<PathBuf>>(); + + let verbose = matches.opt_present("verbose"); + + // User passed in -h/--help? + if matches.opt_present("help") { + usage(0, &opts, verbose, &subcommand_help); + } + + let cmd = match subcommand { + Kind::Build => Subcommand::Build { paths }, + Kind::Check => { + if matches.opt_present("all-targets") { + println!( + "Warning: --all-targets is now on by default and does not need to be passed explicitly." + ); + } + Subcommand::Check { paths } + } + Kind::Clippy => Subcommand::Clippy { + paths, + fix: matches.opt_present("fix"), + clippy_lint_allow: matches.opt_strs("A"), + clippy_lint_warn: matches.opt_strs("W"), + clippy_lint_deny: matches.opt_strs("D"), + clippy_lint_forbid: matches.opt_strs("F"), + }, + Kind::Fix => Subcommand::Fix { paths }, + Kind::Test => Subcommand::Test { + paths, + bless: matches.opt_present("bless"), + force_rerun: matches.opt_present("force-rerun"), + compare_mode: matches.opt_str("compare-mode"), + pass: matches.opt_str("pass"), + run: matches.opt_str("run"), + skip: matches.opt_strs("skip"), + test_args: matches.opt_strs("test-args"), + rustc_args: matches.opt_strs("rustc-args"), + fail_fast: !matches.opt_present("no-fail-fast"), + rustfix_coverage: matches.opt_present("rustfix-coverage"), + doc_tests: if matches.opt_present("doc") { + DocTests::Only + } else if matches.opt_present("no-doc") { + DocTests::No + } else { + DocTests::Yes + }, + }, + Kind::Bench => Subcommand::Bench { paths, test_args: matches.opt_strs("test-args") }, + Kind::Doc => Subcommand::Doc { paths, open: matches.opt_present("open") }, + Kind::Clean => { + if !paths.is_empty() { + println!("\nclean does not take a path argument\n"); + usage(1, &opts, verbose, &subcommand_help); + } + + Subcommand::Clean { all: matches.opt_present("all") } + } + Kind::Format => Subcommand::Format { check: matches.opt_present("check"), paths }, + Kind::Dist => Subcommand::Dist { paths }, + Kind::Install => Subcommand::Install { paths }, + Kind::Run => { + if paths.is_empty() { + println!("\nrun requires at least a path!\n"); + usage(1, &opts, verbose, &subcommand_help); + } + Subcommand::Run { paths } + } + Kind::Setup => { + let profile = if paths.len() > 1 { + println!("\nat most one profile can be passed to setup\n"); + usage(1, &opts, verbose, &subcommand_help) + } else if let Some(path) = paths.pop() { + let profile_string = t!(path.into_os_string().into_string().map_err( + |path| format!("{} is not a valid UTF8 string", path.to_string_lossy()) + )); + + profile_string.parse().unwrap_or_else(|err| { + eprintln!("error: {}", err); + eprintln!("help: the available profiles are:"); + eprint!("{}", Profile::all_for_help("- ")); + crate::detail_exit(1); + }) + } else { + t!(crate::setup::interactive_path()) + }; + Subcommand::Setup { profile } + } + }; + + Flags { + verbose: matches.opt_count("verbose"), + stage: matches.opt_str("stage").map(|j| j.parse().expect("`stage` should be a number")), + dry_run: matches.opt_present("dry-run"), + on_fail: matches.opt_str("on-fail"), + rustc_error_format: matches.opt_str("error-format"), + json_output: matches.opt_present("json-output"), + keep_stage: matches + .opt_strs("keep-stage") + .into_iter() + .map(|j| j.parse().expect("`keep-stage` should be a number")) + .collect(), + keep_stage_std: matches + .opt_strs("keep-stage-std") + .into_iter() + .map(|j| j.parse().expect("`keep-stage-std` should be a number")) + .collect(), + host: if matches.opt_present("host") { + Some( + split(&matches.opt_strs("host")) + .into_iter() + .map(|x| TargetSelection::from_user(&x)) + .collect::<Vec<_>>(), + ) + } else { + None + }, + target: if matches.opt_present("target") { + Some( + split(&matches.opt_strs("target")) + .into_iter() + .map(|x| TargetSelection::from_user(&x)) + .collect::<Vec<_>>(), + ) + } else { + None + }, + config: matches.opt_str("config").map(PathBuf::from), + build_dir: matches.opt_str("build-dir").map(PathBuf::from), + jobs: matches.opt_str("jobs").map(|j| j.parse().expect("`jobs` should be a number")), + cmd, + incremental: matches.opt_present("incremental"), + exclude: split(&matches.opt_strs("exclude")) + .into_iter() + .map(|p| p.into()) + .collect::<Vec<_>>(), + include_default_paths: matches.opt_present("include-default-paths"), + deny_warnings: parse_deny_warnings(&matches), + llvm_skip_rebuild: matches.opt_str("llvm-skip-rebuild").map(|s| s.to_lowercase()).map( + |s| s.parse::<bool>().expect("`llvm-skip-rebuild` should be either true or false"), + ), + color: matches + .opt_get_default("color", Color::Auto) + .expect("`color` should be `always`, `never`, or `auto`"), + rust_profile_use: matches.opt_str("rust-profile-use"), + rust_profile_generate: matches.opt_str("rust-profile-generate"), + llvm_profile_use: matches.opt_str("llvm-profile-use"), + llvm_profile_generate: matches.opt_present("llvm-profile-generate"), + } + } +} + +impl Subcommand { + pub fn kind(&self) -> Kind { + match self { + Subcommand::Bench { .. } => Kind::Bench, + Subcommand::Build { .. } => Kind::Build, + Subcommand::Check { .. } => Kind::Check, + Subcommand::Clippy { .. } => Kind::Clippy, + Subcommand::Doc { .. } => Kind::Doc, + Subcommand::Fix { .. } => Kind::Fix, + Subcommand::Format { .. } => Kind::Format, + Subcommand::Test { .. } => Kind::Test, + Subcommand::Clean { .. } => Kind::Clean, + Subcommand::Dist { .. } => Kind::Dist, + Subcommand::Install { .. } => Kind::Install, + Subcommand::Run { .. } => Kind::Run, + Subcommand::Setup { .. } => Kind::Setup, + } + } + + pub fn test_args(&self) -> Vec<&str> { + let mut args = vec![]; + + match *self { + Subcommand::Test { ref skip, .. } => { + for s in skip { + args.push("--skip"); + args.push(s.as_str()); + } + } + _ => (), + }; + + match *self { + Subcommand::Test { ref test_args, .. } | Subcommand::Bench { ref test_args, .. } => { + args.extend(test_args.iter().flat_map(|s| s.split_whitespace())) + } + _ => (), + } + + args + } + + pub fn rustc_args(&self) -> Vec<&str> { + match *self { + Subcommand::Test { ref rustc_args, .. } => { + rustc_args.iter().flat_map(|s| s.split_whitespace()).collect() + } + _ => Vec::new(), + } + } + + pub fn fail_fast(&self) -> bool { + match *self { + Subcommand::Test { fail_fast, .. } => fail_fast, + _ => false, + } + } + + pub fn doc_tests(&self) -> DocTests { + match *self { + Subcommand::Test { doc_tests, .. } => doc_tests, + _ => DocTests::Yes, + } + } + + pub fn bless(&self) -> bool { + match *self { + Subcommand::Test { bless, .. } => bless, + _ => false, + } + } + + pub fn force_rerun(&self) -> bool { + match *self { + Subcommand::Test { force_rerun, .. } => force_rerun, + _ => false, + } + } + + pub fn rustfix_coverage(&self) -> bool { + match *self { + Subcommand::Test { rustfix_coverage, .. } => rustfix_coverage, + _ => false, + } + } + + pub fn compare_mode(&self) -> Option<&str> { + match *self { + Subcommand::Test { ref compare_mode, .. } => compare_mode.as_ref().map(|s| &s[..]), + _ => None, + } + } + + pub fn pass(&self) -> Option<&str> { + match *self { + Subcommand::Test { ref pass, .. } => pass.as_ref().map(|s| &s[..]), + _ => None, + } + } + + pub fn run(&self) -> Option<&str> { + match *self { + Subcommand::Test { ref run, .. } => run.as_ref().map(|s| &s[..]), + _ => None, + } + } + + pub fn open(&self) -> bool { + match *self { + Subcommand::Doc { open, .. } => open, + _ => false, + } + } +} + +fn split(s: &[String]) -> Vec<String> { + s.iter().flat_map(|s| s.split(',')).filter(|s| !s.is_empty()).map(|s| s.to_string()).collect() +} + +fn parse_deny_warnings(matches: &getopts::Matches) -> Option<bool> { + match matches.opt_str("warnings").as_deref() { + Some("deny") => Some(true), + Some("warn") => Some(false), + Some(value) => { + eprintln!(r#"invalid value for --warnings: {:?}, expected "warn" or "deny""#, value,); + crate::detail_exit(1); + } + None => None, + } +} diff --git a/src/bootstrap/format.rs b/src/bootstrap/format.rs new file mode 100644 index 000000000..37322670e --- /dev/null +++ b/src/bootstrap/format.rs @@ -0,0 +1,175 @@ +//! Runs rustfmt on the repository. + +use crate::builder::Builder; +use crate::util::{output, t}; +use ignore::WalkBuilder; +use std::collections::VecDeque; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::mpsc::SyncSender; + +fn rustfmt(src: &Path, rustfmt: &Path, paths: &[PathBuf], check: bool) -> impl FnMut() { + let mut cmd = Command::new(&rustfmt); + // avoid the submodule config paths from coming into play, + // we only allow a single global config for the workspace for now + cmd.arg("--config-path").arg(&src.canonicalize().unwrap()); + cmd.arg("--edition").arg("2021"); + cmd.arg("--unstable-features"); + cmd.arg("--skip-children"); + if check { + cmd.arg("--check"); + } + cmd.args(paths); + let cmd_debug = format!("{:?}", cmd); + let mut cmd = cmd.spawn().expect("running rustfmt"); + // poor man's async: return a closure that'll wait for rustfmt's completion + move || { + let status = cmd.wait().unwrap(); + if !status.success() { + eprintln!( + "Running `{}` failed.\nIf you're running `tidy`, \ + try again with `--bless`. Or, if you just want to format \ + code, run `./x.py fmt` instead.", + cmd_debug, + ); + crate::detail_exit(1); + } + } +} + +#[derive(serde::Deserialize)] +struct RustfmtConfig { + ignore: Vec<String>, +} + +pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) { + if build.config.dry_run { + return; + } + let mut builder = ignore::types::TypesBuilder::new(); + builder.add_defaults(); + builder.select("rust"); + let matcher = builder.build().unwrap(); + let rustfmt_config = build.src.join("rustfmt.toml"); + if !rustfmt_config.exists() { + eprintln!("Not running formatting checks; rustfmt.toml does not exist."); + eprintln!("This may happen in distributed tarballs."); + return; + } + let rustfmt_config = t!(std::fs::read_to_string(&rustfmt_config)); + let rustfmt_config: RustfmtConfig = t!(toml::from_str(&rustfmt_config)); + let mut ignore_fmt = ignore::overrides::OverrideBuilder::new(&build.src); + for ignore in rustfmt_config.ignore { + ignore_fmt.add(&format!("!{}", ignore)).expect(&ignore); + } + let git_available = match Command::new("git") + .arg("--version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + { + Ok(status) => status.success(), + Err(_) => false, + }; + if git_available { + let in_working_tree = match build + .config + .git() + .arg("rev-parse") + .arg("--is-inside-work-tree") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + { + Ok(status) => status.success(), + Err(_) => false, + }; + if in_working_tree { + let untracked_paths_output = output( + build.config.git().arg("status").arg("--porcelain").arg("--untracked-files=normal"), + ); + let untracked_paths = untracked_paths_output + .lines() + .filter(|entry| entry.starts_with("??")) + .map(|entry| { + entry.split(' ').nth(1).expect("every git status entry should list a path") + }); + for untracked_path in untracked_paths { + println!("skip untracked path {} during rustfmt invocations", untracked_path); + // The leading `/` makes it an exact match against the + // repository root, rather than a glob. Without that, if you + // have `foo.rs` in the repository root it will also match + // against anything like `compiler/rustc_foo/src/foo.rs`, + // preventing the latter from being formatted. + ignore_fmt.add(&format!("!/{}", untracked_path)).expect(&untracked_path); + } + } else { + println!("Not in git tree. Skipping git-aware format checks"); + } + } else { + println!("Could not find usable git. Skipping git-aware format checks"); + } + let ignore_fmt = ignore_fmt.build().unwrap(); + + let rustfmt_path = build.initial_rustfmt().unwrap_or_else(|| { + eprintln!("./x.py fmt is not supported on this channel"); + crate::detail_exit(1); + }); + assert!(rustfmt_path.exists(), "{}", rustfmt_path.display()); + let src = build.src.clone(); + let (tx, rx): (SyncSender<PathBuf>, _) = std::sync::mpsc::sync_channel(128); + let walker = match paths.get(0) { + Some(first) => { + let mut walker = WalkBuilder::new(first); + for path in &paths[1..] { + walker.add(path); + } + walker + } + None => WalkBuilder::new(src.clone()), + } + .types(matcher) + .overrides(ignore_fmt) + .build_parallel(); + + // there is a lot of blocking involved in spawning a child process and reading files to format. + // spawn more processes than available concurrency to keep the CPU busy + let max_processes = build.jobs() as usize * 2; + + // spawn child processes on a separate thread so we can batch entries we have received from ignore + let thread = std::thread::spawn(move || { + let mut children = VecDeque::new(); + while let Ok(path) = rx.recv() { + // try getting a few more paths from the channel to amortize the overhead of spawning processes + let paths: Vec<_> = rx.try_iter().take(7).chain(std::iter::once(path)).collect(); + + let child = rustfmt(&src, &rustfmt_path, paths.as_slice(), check); + children.push_back(child); + + if children.len() >= max_processes { + // await oldest child + children.pop_front().unwrap()(); + } + } + + // await remaining children + for mut child in children { + child(); + } + }); + + walker.run(|| { + let tx = tx.clone(); + Box::new(move |entry| { + let entry = t!(entry); + if entry.file_type().map_or(false, |t| t.is_file()) { + t!(tx.send(entry.into_path())); + } + ignore::WalkState::Continue + }) + }); + + drop(tx); + + thread.join().unwrap(); +} diff --git a/src/bootstrap/install.rs b/src/bootstrap/install.rs new file mode 100644 index 000000000..6e49f39ff --- /dev/null +++ b/src/bootstrap/install.rs @@ -0,0 +1,292 @@ +//! Implementation of the install aspects of the compiler. +//! +//! This module is responsible for installing the standard library, +//! compiler, and documentation. + +use std::env; +use std::fs; +use std::path::{Component, Path, PathBuf}; +use std::process::Command; + +use crate::util::t; + +use crate::dist; +use crate::tarball::GeneratedTarball; +use crate::Compiler; + +use crate::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::config::{Config, TargetSelection}; + +#[cfg(target_os = "illumos")] +const SHELL: &str = "bash"; +#[cfg(not(target_os = "illumos"))] +const SHELL: &str = "sh"; + +// We have to run a few shell scripts, which choke quite a bit on both `\` +// characters and on `C:\` paths, so normalize both of them away. +fn sanitize_sh(path: &Path) -> String { + let path = path.to_str().unwrap().replace("\\", "/"); + return change_drive(unc_to_lfs(&path)).unwrap_or(path); + + fn unc_to_lfs(s: &str) -> &str { + s.strip_prefix("//?/").unwrap_or(s) + } + + fn change_drive(s: &str) -> Option<String> { + let mut ch = s.chars(); + let drive = ch.next().unwrap_or('C'); + if ch.next() != Some(':') { + return None; + } + if ch.next() != Some('/') { + return None; + } + Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..])) + } +} + +fn install_sh( + builder: &Builder<'_>, + package: &str, + stage: u32, + host: Option<TargetSelection>, + tarball: &GeneratedTarball, +) { + builder.info(&format!("Install {} stage{} ({:?})", package, stage, host)); + + let prefix = default_path(&builder.config.prefix, "/usr/local"); + let sysconfdir = prefix.join(default_path(&builder.config.sysconfdir, "/etc")); + let datadir = prefix.join(default_path(&builder.config.datadir, "share")); + let docdir = prefix.join(default_path(&builder.config.docdir, "share/doc/rust")); + let mandir = prefix.join(default_path(&builder.config.mandir, "share/man")); + let libdir = prefix.join(default_path(&builder.config.libdir, "lib")); + let bindir = prefix.join(&builder.config.bindir); // Default in config.rs + + let empty_dir = builder.out.join("tmp/empty_dir"); + t!(fs::create_dir_all(&empty_dir)); + + let mut cmd = Command::new(SHELL); + cmd.current_dir(&empty_dir) + .arg(sanitize_sh(&tarball.decompressed_output().join("install.sh"))) + .arg(format!("--prefix={}", prepare_dir(prefix))) + .arg(format!("--sysconfdir={}", prepare_dir(sysconfdir))) + .arg(format!("--datadir={}", prepare_dir(datadir))) + .arg(format!("--docdir={}", prepare_dir(docdir))) + .arg(format!("--bindir={}", prepare_dir(bindir))) + .arg(format!("--libdir={}", prepare_dir(libdir))) + .arg(format!("--mandir={}", prepare_dir(mandir))) + .arg("--disable-ldconfig"); + builder.run(&mut cmd); + t!(fs::remove_dir_all(&empty_dir)); +} + +fn default_path(config: &Option<PathBuf>, default: &str) -> PathBuf { + config.as_ref().cloned().unwrap_or_else(|| PathBuf::from(default)) +} + +fn prepare_dir(mut path: PathBuf) -> String { + // The DESTDIR environment variable is a standard way to install software in a subdirectory + // while keeping the original directory structure, even if the prefix or other directories + // contain absolute paths. + // + // More information on the environment variable is available here: + // https://www.gnu.org/prep/standards/html_node/DESTDIR.html + if let Some(destdir) = env::var_os("DESTDIR").map(PathBuf::from) { + let without_destdir = path.clone(); + path = destdir; + // Custom .join() which ignores disk roots. + for part in without_destdir.components() { + if let Component::Normal(s) = part { + path.push(s) + } + } + } + + // The installation command is not executed from the current directory, but from a temporary + // directory. To prevent relative paths from breaking this converts relative paths to absolute + // paths. std::fs::canonicalize is not used as that requires the path to actually be present. + if path.is_relative() { + path = std::env::current_dir().expect("failed to get the current directory").join(path); + assert!(path.is_absolute(), "could not make the path relative"); + } + + sanitize_sh(&path) +} + +macro_rules! install { + (($sel:ident, $builder:ident, $_config:ident), + $($name:ident, + $condition_name: ident = $path_or_alias: literal, + $default_cond:expr, + only_hosts: $only_hosts:expr, + $run_item:block $(, $c:ident)*;)+) => { + $( + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] + pub struct $name { + pub compiler: Compiler, + pub target: TargetSelection, + } + + impl $name { + #[allow(dead_code)] + fn should_build(config: &Config) -> bool { + config.extended && config.tools.as_ref() + .map_or(true, |t| t.contains($path_or_alias)) + } + } + + impl Step for $name { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = $only_hosts; + $(const $c: bool = true;)* + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let $_config = &run.builder.config; + run.$condition_name($path_or_alias).default_condition($default_cond) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure($name { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + fn run($sel, $builder: &Builder<'_>) { + $run_item + } + })+ + } +} + +install!((self, builder, _config), + Docs, path = "src/doc", _config.docs, only_hosts: false, { + let tarball = builder.ensure(dist::Docs { host: self.target }).expect("missing docs"); + install_sh(builder, "docs", self.compiler.stage, Some(self.target), &tarball); + }; + Std, path = "library/std", true, only_hosts: false, { + for target in &builder.targets { + // `expect` should be safe, only None when host != build, but this + // only runs when host == build + let tarball = builder.ensure(dist::Std { + compiler: self.compiler, + target: *target + }).expect("missing std"); + install_sh(builder, "std", self.compiler.stage, Some(*target), &tarball); + } + }; + Cargo, alias = "cargo", Self::should_build(_config), only_hosts: true, { + let tarball = builder + .ensure(dist::Cargo { compiler: self.compiler, target: self.target }) + .expect("missing cargo"); + install_sh(builder, "cargo", self.compiler.stage, Some(self.target), &tarball); + }; + Rls, alias = "rls", Self::should_build(_config), only_hosts: true, { + if let Some(tarball) = builder.ensure(dist::Rls { compiler: self.compiler, target: self.target }) { + install_sh(builder, "rls", self.compiler.stage, Some(self.target), &tarball); + } else { + builder.info( + &format!("skipping Install RLS stage{} ({})", self.compiler.stage, self.target), + ); + } + }; + RustAnalyzer, alias = "rust-analyzer", Self::should_build(_config), only_hosts: true, { + if let Some(tarball) = + builder.ensure(dist::RustAnalyzer { compiler: self.compiler, target: self.target }) + { + install_sh(builder, "rust-analyzer", self.compiler.stage, Some(self.target), &tarball); + } else { + builder.info( + &format!("skipping Install rust-analyzer stage{} ({})", self.compiler.stage, self.target), + ); + } + }; + Clippy, alias = "clippy", Self::should_build(_config), only_hosts: true, { + let tarball = builder + .ensure(dist::Clippy { compiler: self.compiler, target: self.target }) + .expect("missing clippy"); + install_sh(builder, "clippy", self.compiler.stage, Some(self.target), &tarball); + }; + Miri, alias = "miri", Self::should_build(_config), only_hosts: true, { + if let Some(tarball) = builder.ensure(dist::Miri { compiler: self.compiler, target: self.target }) { + install_sh(builder, "miri", self.compiler.stage, Some(self.target), &tarball); + } else { + builder.info( + &format!("skipping Install miri stage{} ({})", self.compiler.stage, self.target), + ); + } + }; + Rustfmt, alias = "rustfmt", Self::should_build(_config), only_hosts: true, { + if let Some(tarball) = builder.ensure(dist::Rustfmt { + compiler: self.compiler, + target: self.target + }) { + install_sh(builder, "rustfmt", self.compiler.stage, Some(self.target), &tarball); + } else { + builder.info( + &format!("skipping Install Rustfmt stage{} ({})", self.compiler.stage, self.target), + ); + } + }; + RustDemangler, alias = "rust-demangler", Self::should_build(_config), only_hosts: true, { + // Note: Even though `should_build` may return true for `extended` default tools, + // dist::RustDemangler may still return None, unless the target-dependent `profiler` config + // is also true, or the `tools` array explicitly includes "rust-demangler". + if let Some(tarball) = builder.ensure(dist::RustDemangler { + compiler: self.compiler, + target: self.target + }) { + install_sh(builder, "rust-demangler", self.compiler.stage, Some(self.target), &tarball); + } else { + builder.info( + &format!("skipping Install RustDemangler stage{} ({})", + self.compiler.stage, self.target), + ); + } + }; + Analysis, alias = "analysis", Self::should_build(_config), only_hosts: false, { + // `expect` should be safe, only None with host != build, but this + // only uses the `build` compiler + let tarball = builder.ensure(dist::Analysis { + // Find the actual compiler (handling the full bootstrap option) which + // produced the save-analysis data because that data isn't copied + // through the sysroot uplifting. + compiler: builder.compiler_for(builder.top_stage, builder.config.build, self.target), + target: self.target + }).expect("missing analysis"); + install_sh(builder, "analysis", self.compiler.stage, Some(self.target), &tarball); + }; + Rustc, path = "compiler/rustc", true, only_hosts: true, { + let tarball = builder.ensure(dist::Rustc { + compiler: builder.compiler(builder.top_stage, self.target), + }); + install_sh(builder, "rustc", self.compiler.stage, Some(self.target), &tarball); + }; +); + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Src { + pub stage: u32, +} + +impl Step for Src { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let config = &run.builder.config; + let cond = config.extended && config.tools.as_ref().map_or(true, |t| t.contains("src")); + run.path("src").default_condition(cond) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Src { stage: run.builder.top_stage }); + } + + fn run(self, builder: &Builder<'_>) { + let tarball = builder.ensure(dist::Src); + install_sh(builder, "src", self.stage, None, &tarball); + } +} diff --git a/src/bootstrap/job.rs b/src/bootstrap/job.rs new file mode 100644 index 000000000..5c0322e18 --- /dev/null +++ b/src/bootstrap/job.rs @@ -0,0 +1,140 @@ +//! Job management on Windows for bootstrapping +//! +//! Most of the time when you're running a build system (e.g., make) you expect +//! Ctrl-C or abnormal termination to actually terminate the entire tree of +//! process in play, not just the one at the top. This currently works "by +//! default" on Unix platforms because Ctrl-C actually sends a signal to the +//! *process group* rather than the parent process, so everything will get torn +//! down. On Windows, however, this does not happen and Ctrl-C just kills the +//! parent process. +//! +//! To achieve the same semantics on Windows we use Job Objects to ensure that +//! all processes die at the same time. Job objects have a mode of operation +//! where when all handles to the object are closed it causes all child +//! processes associated with the object to be terminated immediately. +//! Conveniently whenever a process in the job object spawns a new process the +//! child will be associated with the job object as well. This means if we add +//! ourselves to the job object we create then everything will get torn down! +//! +//! Unfortunately most of the time the build system is actually called from a +//! python wrapper (which manages things like building the build system) so this +//! all doesn't quite cut it so far. To go the last mile we duplicate the job +//! object handle into our parent process (a python process probably) and then +//! close our own handle. This means that the only handle to the job object +//! resides in the parent python process, so when python dies the whole build +//! system dies (as one would probably expect!). +//! +//! Note that this module has a #[cfg(windows)] above it as none of this logic +//! is required on Unix. + +#![allow(nonstandard_style, dead_code)] + +use crate::Build; +use std::env; +use std::io; +use std::mem; +use std::ptr; + +use winapi::shared::minwindef::{DWORD, FALSE, LPVOID}; +use winapi::um::errhandlingapi::SetErrorMode; +use winapi::um::handleapi::{CloseHandle, DuplicateHandle}; +use winapi::um::jobapi2::{AssignProcessToJobObject, CreateJobObjectW, SetInformationJobObject}; +use winapi::um::processthreadsapi::{GetCurrentProcess, OpenProcess}; +use winapi::um::winbase::{BELOW_NORMAL_PRIORITY_CLASS, SEM_NOGPFAULTERRORBOX}; +use winapi::um::winnt::{ + JobObjectExtendedLimitInformation, DUPLICATE_SAME_ACCESS, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, + JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOB_OBJECT_LIMIT_PRIORITY_CLASS, PROCESS_DUP_HANDLE, +}; + +pub unsafe fn setup(build: &mut Build) { + // Enable the Windows Error Reporting dialog which msys disables, + // so we can JIT debug rustc + let mode = SetErrorMode(0); + SetErrorMode(mode & !SEM_NOGPFAULTERRORBOX); + + // Create a new job object for us to use + let job = CreateJobObjectW(ptr::null_mut(), ptr::null()); + assert!(!job.is_null(), "{}", io::Error::last_os_error()); + + // Indicate that when all handles to the job object are gone that all + // process in the object should be killed. Note that this includes our + // entire process tree by default because we've added ourselves and our + // children will reside in the job by default. + let mut info = mem::zeroed::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>(); + info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + if build.config.low_priority { + info.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_PRIORITY_CLASS; + info.BasicLimitInformation.PriorityClass = BELOW_NORMAL_PRIORITY_CLASS; + } + let r = SetInformationJobObject( + job, + JobObjectExtendedLimitInformation, + &mut info as *mut _ as LPVOID, + mem::size_of_val(&info) as DWORD, + ); + assert!(r != 0, "{}", io::Error::last_os_error()); + + // Assign our process to this job object. Note that if this fails, one very + // likely reason is that we are ourselves already in a job object! This can + // happen on the build bots that we've got for Windows, or if just anyone + // else is instrumenting the build. In this case we just bail out + // immediately and assume that they take care of it. + // + // Also note that nested jobs (why this might fail) are supported in recent + // versions of Windows, but the version of Windows that our bots are running + // at least don't support nested job objects. + let r = AssignProcessToJobObject(job, GetCurrentProcess()); + if r == 0 { + CloseHandle(job); + return; + } + + // If we've got a parent process (e.g., the python script that called us) + // then move ownership of this job object up to them. That way if the python + // script is killed (e.g., via ctrl-c) then we'll all be torn down. + // + // If we don't have a parent (e.g., this was run directly) then we + // intentionally leak the job object handle. When our process exits + // (normally or abnormally) it will close the handle implicitly, causing all + // processes in the job to be cleaned up. + let pid = match env::var("BOOTSTRAP_PARENT_ID") { + Ok(s) => s, + Err(..) => return, + }; + + let parent = OpenProcess(PROCESS_DUP_HANDLE, FALSE, pid.parse().unwrap()); + + // If we get a null parent pointer here, it is possible that either + // we have got an invalid pid or the parent process has been closed. + // Since the first case rarely happens + // (only when wrongly setting the environmental variable), + // so it might be better to improve the experience of the second case + // when users have interrupted the parent process and we don't finish + // duplicating the handle yet. + // We just need close the job object if that occurs. + if parent.is_null() { + CloseHandle(job); + return; + } + + let mut parent_handle = ptr::null_mut(); + let r = DuplicateHandle( + GetCurrentProcess(), + job, + parent, + &mut parent_handle, + 0, + FALSE, + DUPLICATE_SAME_ACCESS, + ); + + // If this failed, well at least we tried! An example of DuplicateHandle + // failing in the past has been when the wrong python2 package spawned this + // build system (e.g., the `python2` package in MSYS instead of + // `mingw-w64-x86_64-python2`. Not sure why it failed, but the "failure + // mode" here is that we only clean everything up when the build system + // dies, not when the python parent does, so not too bad. + if r != 0 { + CloseHandle(job); + } +} diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs new file mode 100644 index 000000000..d265277b4 --- /dev/null +++ b/src/bootstrap/lib.rs @@ -0,0 +1,1679 @@ +//! Implementation of rustbuild, the Rust build system. +//! +//! This module, and its descendants, are the implementation of the Rust build +//! system. Most of this build system is backed by Cargo but the outer layer +//! here serves as the ability to orchestrate calling Cargo, sequencing Cargo +//! builds, building artifacts like LLVM, etc. The goals of rustbuild are: +//! +//! * To be an easily understandable, easily extensible, and maintainable build +//! system. +//! * Leverage standard tools in the Rust ecosystem to build the compiler, aka +//! crates.io and Cargo. +//! * A standard interface to build across all platforms, including MSVC +//! +//! ## Architecture +//! +//! The build system defers most of the complicated logic managing invocations +//! of rustc and rustdoc to Cargo itself. However, moving through various stages +//! and copying artifacts is still necessary for it to do. Each time rustbuild +//! is invoked, it will iterate through the list of predefined steps and execute +//! each serially in turn if it matches the paths passed or is a default rule. +//! For each step rustbuild relies on the step internally being incremental and +//! parallel. Note, though, that the `-j` parameter to rustbuild gets forwarded +//! to appropriate test harnesses and such. +//! +//! Most of the "meaty" steps that matter are backed by Cargo, which does indeed +//! have its own parallelism and incremental management. Later steps, like +//! tests, aren't incremental and simply run the entire suite currently. +//! However, compiletest itself tries to avoid running tests when the artifacts +//! that are involved (mainly the compiler) haven't changed. +//! +//! When you execute `x.py build`, the steps executed are: +//! +//! * First, the python script is run. This will automatically download the +//! stage0 rustc and cargo according to `src/stage0.json`, or use the cached +//! versions if they're available. These are then used to compile rustbuild +//! itself (using Cargo). Finally, control is then transferred to rustbuild. +//! +//! * Rustbuild takes over, performs sanity checks, probes the environment, +//! reads configuration, and starts executing steps as it reads the command +//! line arguments (paths) or going through the default rules. +//! +//! The build output will be something like the following: +//! +//! Building stage0 std artifacts +//! Copying stage0 std +//! Building stage0 test artifacts +//! Copying stage0 test +//! Building stage0 compiler artifacts +//! Copying stage0 rustc +//! Assembling stage1 compiler +//! Building stage1 std artifacts +//! Copying stage1 std +//! Building stage1 test artifacts +//! Copying stage1 test +//! Building stage1 compiler artifacts +//! Copying stage1 rustc +//! Assembling stage2 compiler +//! Uplifting stage1 std +//! Uplifting stage1 test +//! Uplifting stage1 rustc +//! +//! Let's disect that a little: +//! +//! ## Building stage0 {std,test,compiler} artifacts +//! +//! These steps use the provided (downloaded, usually) compiler to compile the +//! local Rust source into libraries we can use. +//! +//! ## Copying stage0 {std,test,rustc} +//! +//! This copies the build output from Cargo into +//! `build/$HOST/stage0-sysroot/lib/rustlib/$ARCH/lib`. FIXME: this step's +//! documentation should be expanded -- the information already here may be +//! incorrect. +//! +//! ## Assembling stage1 compiler +//! +//! This copies the libraries we built in "building stage0 ... artifacts" into +//! the stage1 compiler's lib directory. These are the host libraries that the +//! compiler itself uses to run. These aren't actually used by artifacts the new +//! compiler generates. This step also copies the rustc and rustdoc binaries we +//! generated into build/$HOST/stage/bin. +//! +//! The stage1/bin/rustc is a fully functional compiler, but it doesn't yet have +//! any libraries to link built binaries or libraries to. The next 3 steps will +//! provide those libraries for it; they are mostly equivalent to constructing +//! the stage1/bin compiler so we don't go through them individually. +//! +//! ## Uplifting stage1 {std,test,rustc} +//! +//! This step copies the libraries from the stage1 compiler sysroot into the +//! stage2 compiler. This is done to avoid rebuilding the compiler; libraries +//! we'd build in this step should be identical (in function, if not necessarily +//! identical on disk) so there's no need to recompile the compiler again. Note +//! that if you want to, you can enable the full-bootstrap option to change this +//! behavior. +//! +//! Each step is driven by a separate Cargo project and rustbuild orchestrates +//! copying files between steps and otherwise preparing for Cargo to run. +//! +//! ## Further information +//! +//! More documentation can be found in each respective module below, and you can +//! also check out the `src/bootstrap/README.md` file for more information. + +use std::cell::{Cell, RefCell}; +use std::collections::{HashMap, HashSet}; +use std::env; +use std::fs::{self, File}; +use std::io; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::str; + +use filetime::FileTime; +use once_cell::sync::OnceCell; + +use crate::builder::Kind; +use crate::config::{LlvmLibunwind, TargetSelection}; +use crate::util::{ + check_run, exe, libdir, mtime, output, run, run_suppressed, try_run, try_run_suppressed, CiEnv, +}; + +mod builder; +mod cache; +mod cc_detect; +mod channel; +mod check; +mod clean; +mod compile; +mod config; +mod dist; +mod doc; +mod flags; +mod format; +mod install; +mod metadata; +mod native; +mod run; +mod sanity; +mod setup; +mod tarball; +mod test; +mod tool; +mod toolstate; +pub mod util; + +#[cfg(feature = "build-metrics")] +mod metrics; + +#[cfg(windows)] +mod job; + +#[cfg(all(unix, not(target_os = "haiku")))] +mod job { + pub unsafe fn setup(build: &mut crate::Build) { + if build.config.low_priority { + libc::setpriority(libc::PRIO_PGRP as _, 0, 10); + } + } +} + +#[cfg(any(target_os = "haiku", target_os = "hermit", not(any(unix, windows))))] +mod job { + pub unsafe fn setup(_build: &mut crate::Build) {} +} + +pub use crate::builder::PathSet; +use crate::cache::{Interned, INTERNER}; +pub use crate::config::Config; +pub use crate::flags::Subcommand; + +const LLVM_TOOLS: &[&str] = &[ + "llvm-cov", // used to generate coverage report + "llvm-nm", // used to inspect binaries; it shows symbol names, their sizes and visibility + "llvm-objcopy", // used to transform ELFs into binary format which flashing tools consume + "llvm-objdump", // used to disassemble programs + "llvm-profdata", // used to inspect and merge files generated by profiles + "llvm-readobj", // used to get information from ELFs/objects that the other tools don't provide + "llvm-size", // used to prints the size of the linker sections of a program + "llvm-strip", // used to discard symbols from binary files to reduce their size + "llvm-ar", // used for creating and modifying archive files + "llvm-as", // used to convert LLVM assembly to LLVM bitcode + "llvm-dis", // used to disassemble LLVM bitcode + "llc", // used to compile LLVM bytecode + "opt", // used to optimize LLVM bytecode +]; + +pub const VERSION: usize = 2; + +/// Extra --check-cfg to add when building +/// (Mode restriction, config name, config values (if any)) +const EXTRA_CHECK_CFGS: &[(Option<Mode>, &'static str, Option<&[&'static str]>)] = &[ + (None, "bootstrap", None), + (Some(Mode::Rustc), "parallel_compiler", None), + (Some(Mode::ToolRustc), "parallel_compiler", None), + (Some(Mode::Std), "stdarch_intel_sde", None), + (Some(Mode::Std), "no_fp_fmt_parse", None), + (Some(Mode::Std), "no_global_oom_handling", None), + (Some(Mode::Std), "freebsd12", None), + (Some(Mode::Std), "backtrace_in_libstd", None), + /* Extra values not defined in the built-in targets yet, but used in std */ + (Some(Mode::Std), "target_env", Some(&["libnx"])), + (Some(Mode::Std), "target_os", Some(&["watchos"])), + ( + Some(Mode::Std), + "target_arch", + Some(&["asmjs", "spirv", "nvptx", "nvptx64", "le32", "xtensa"]), + ), + /* Extra names used by dependencies */ + // FIXME: Used by rustfmt is their test but is invalid (neither cargo nor bootstrap ever set + // this config) should probably by removed or use a allow attribute. + (Some(Mode::ToolRustc), "release", None), + // FIXME: Used by stdarch in their test, should use a allow attribute instead. + (Some(Mode::Std), "dont_compile_me", None), + // FIXME: Used by serde_json, but we should not be triggering on external dependencies. + (Some(Mode::Rustc), "no_btreemap_remove_entry", None), + (Some(Mode::ToolRustc), "no_btreemap_remove_entry", None), + // FIXME: Used by crossbeam-utils, but we should not be triggering on external dependencies. + (Some(Mode::Rustc), "crossbeam_loom", None), + (Some(Mode::ToolRustc), "crossbeam_loom", None), + // FIXME: Used by proc-macro2, but we should not be triggering on external dependencies. + (Some(Mode::Rustc), "span_locations", None), + (Some(Mode::ToolRustc), "span_locations", None), +]; + +/// A structure representing a Rust compiler. +/// +/// Each compiler has a `stage` that it is associated with and a `host` that +/// corresponds to the platform the compiler runs on. This structure is used as +/// a parameter to many methods below. +#[derive(Eq, PartialOrd, Ord, PartialEq, Clone, Copy, Hash, Debug)] +pub struct Compiler { + stage: u32, + host: TargetSelection, +} + +#[derive(PartialEq, Eq, Copy, Clone, Debug)] +pub enum DocTests { + /// Run normal tests and doc tests (default). + Yes, + /// Do not run any doc tests. + No, + /// Only run doc tests. + Only, +} + +pub enum GitRepo { + Rustc, + Llvm, +} + +/// Global configuration for the build system. +/// +/// This structure transitively contains all configuration for the build system. +/// All filesystem-encoded configuration is in `config`, all flags are in +/// `flags`, and then parsed or probed information is listed in the keys below. +/// +/// This structure is a parameter of almost all methods in the build system, +/// although most functions are implemented as free functions rather than +/// methods specifically on this structure itself (to make it easier to +/// organize). +pub struct Build { + /// User-specified configuration from `config.toml`. + config: Config, + + // Version information + version: String, + + // Properties derived from the above configuration + src: PathBuf, + out: PathBuf, + bootstrap_out: PathBuf, + rust_info: channel::GitInfo, + cargo_info: channel::GitInfo, + rls_info: channel::GitInfo, + rust_analyzer_info: channel::GitInfo, + clippy_info: channel::GitInfo, + miri_info: channel::GitInfo, + rustfmt_info: channel::GitInfo, + in_tree_llvm_info: channel::GitInfo, + local_rebuild: bool, + fail_fast: bool, + doc_tests: DocTests, + verbosity: usize, + + // Targets for which to build + build: TargetSelection, + hosts: Vec<TargetSelection>, + targets: Vec<TargetSelection>, + + initial_rustc: PathBuf, + initial_cargo: PathBuf, + initial_lld: PathBuf, + initial_libdir: PathBuf, + + // Runtime state filled in later on + // C/C++ compilers and archiver for all targets + cc: HashMap<TargetSelection, cc::Tool>, + cxx: HashMap<TargetSelection, cc::Tool>, + ar: HashMap<TargetSelection, PathBuf>, + ranlib: HashMap<TargetSelection, PathBuf>, + // Miscellaneous + // allow bidirectional lookups: both name -> path and path -> name + crates: HashMap<Interned<String>, Crate>, + crate_paths: HashMap<PathBuf, Interned<String>>, + is_sudo: bool, + ci_env: CiEnv, + delayed_failures: RefCell<Vec<String>>, + prerelease_version: Cell<Option<u32>>, + tool_artifacts: + RefCell<HashMap<TargetSelection, HashMap<String, (&'static str, PathBuf, Vec<String>)>>>, + + #[cfg(feature = "build-metrics")] + metrics: metrics::BuildMetrics, +} + +#[derive(Debug)] +struct Crate { + name: Interned<String>, + deps: HashSet<Interned<String>>, + path: PathBuf, +} + +impl Crate { + fn local_path(&self, build: &Build) -> PathBuf { + self.path.strip_prefix(&build.config.src).unwrap().into() + } +} + +/// When building Rust various objects are handled differently. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum DependencyType { + /// Libraries originating from proc-macros. + Host, + /// Typical Rust libraries. + Target, + /// Non Rust libraries and objects shipped to ease usage of certain targets. + TargetSelfContained, +} + +/// The various "modes" of invoking Cargo. +/// +/// These entries currently correspond to the various output directories of the +/// build system, with each mod generating output in a different directory. +#[derive(Debug, Hash, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Mode { + /// Build the standard library, placing output in the "stageN-std" directory. + Std, + + /// Build librustc, and compiler libraries, placing output in the "stageN-rustc" directory. + Rustc, + + /// Build a codegen backend for rustc, placing the output in the "stageN-codegen" directory. + Codegen, + + /// Build a tool, placing output in the "stage0-bootstrap-tools" + /// directory. This is for miscellaneous sets of tools that are built + /// using the bootstrap stage0 compiler in its entirety (target libraries + /// and all). Typically these tools compile with stable Rust. + ToolBootstrap, + + /// Build a tool which uses the locally built std, placing output in the + /// "stageN-tools" directory. Its usage is quite rare, mainly used by + /// compiletest which needs libtest. + ToolStd, + + /// Build a tool which uses the locally built rustc and the target std, + /// placing the output in the "stageN-tools" directory. This is used for + /// anything that needs a fully functional rustc, such as rustdoc, clippy, + /// cargo, rls, rustfmt, miri, etc. + ToolRustc, +} + +impl Mode { + pub fn is_tool(&self) -> bool { + matches!(self, Mode::ToolBootstrap | Mode::ToolRustc | Mode::ToolStd) + } + + pub fn must_support_dlopen(&self) -> bool { + matches!(self, Mode::Std | Mode::Codegen) + } +} + +pub enum CLang { + C, + Cxx, +} + +impl Build { + /// Creates a new set of build configuration from the `flags` on the command + /// line and the filesystem `config`. + /// + /// By default all build output will be placed in the current directory. + pub fn new(config: Config) -> Build { + let src = config.src.clone(); + let out = config.out.clone(); + + #[cfg(unix)] + // keep this consistent with the equivalent check in x.py: + // https://github.com/rust-lang/rust/blob/a8a33cf27166d3eabaffc58ed3799e054af3b0c6/src/bootstrap/bootstrap.py#L796-L797 + let is_sudo = match env::var_os("SUDO_USER") { + Some(_sudo_user) => { + let uid = unsafe { libc::getuid() }; + uid == 0 + } + None => false, + }; + #[cfg(not(unix))] + let is_sudo = false; + + let ignore_git = config.ignore_git; + let rust_info = channel::GitInfo::new(ignore_git, &src); + let cargo_info = channel::GitInfo::new(ignore_git, &src.join("src/tools/cargo")); + let rls_info = channel::GitInfo::new(ignore_git, &src.join("src/tools/rls")); + let rust_analyzer_info = + channel::GitInfo::new(ignore_git, &src.join("src/tools/rust-analyzer")); + let clippy_info = channel::GitInfo::new(ignore_git, &src.join("src/tools/clippy")); + let miri_info = channel::GitInfo::new(ignore_git, &src.join("src/tools/miri")); + let rustfmt_info = channel::GitInfo::new(ignore_git, &src.join("src/tools/rustfmt")); + + // we always try to use git for LLVM builds + let in_tree_llvm_info = channel::GitInfo::new(false, &src.join("src/llvm-project")); + + let initial_target_libdir_str = if config.dry_run { + "/dummy/lib/path/to/lib/".to_string() + } else { + output( + Command::new(&config.initial_rustc) + .arg("--target") + .arg(config.build.rustc_target_arg()) + .arg("--print") + .arg("target-libdir"), + ) + }; + let initial_target_dir = Path::new(&initial_target_libdir_str).parent().unwrap(); + let initial_lld = initial_target_dir.join("bin").join("rust-lld"); + + let initial_sysroot = if config.dry_run { + "/dummy".to_string() + } else { + output(Command::new(&config.initial_rustc).arg("--print").arg("sysroot")) + }; + let initial_libdir = initial_target_dir + .parent() + .unwrap() + .parent() + .unwrap() + .strip_prefix(initial_sysroot.trim()) + .unwrap() + .to_path_buf(); + + let version = std::fs::read_to_string(src.join("src").join("version")) + .expect("failed to read src/version"); + let version = version.trim(); + + let bootstrap_out = if std::env::var("BOOTSTRAP_PYTHON").is_ok() { + out.join("bootstrap").join("debug") + } else { + let workspace_target_dir = std::env::var("CARGO_TARGET_DIR") + .map(PathBuf::from) + .unwrap_or_else(|_| src.join("target")); + let bootstrap_out = workspace_target_dir.join("debug"); + if !bootstrap_out.join("rustc").exists() && !cfg!(test) { + // this restriction can be lifted whenever https://github.com/rust-lang/rfcs/pull/3028 is implemented + panic!("run `cargo build --bins` before `cargo run`") + } + bootstrap_out + }; + + let mut build = Build { + initial_rustc: config.initial_rustc.clone(), + initial_cargo: config.initial_cargo.clone(), + initial_lld, + initial_libdir, + local_rebuild: config.local_rebuild, + fail_fast: config.cmd.fail_fast(), + doc_tests: config.cmd.doc_tests(), + verbosity: config.verbose, + + build: config.build, + hosts: config.hosts.clone(), + targets: config.targets.clone(), + + config, + version: version.to_string(), + src, + out, + bootstrap_out, + + rust_info, + cargo_info, + rls_info, + rust_analyzer_info, + clippy_info, + miri_info, + rustfmt_info, + in_tree_llvm_info, + cc: HashMap::new(), + cxx: HashMap::new(), + ar: HashMap::new(), + ranlib: HashMap::new(), + crates: HashMap::new(), + crate_paths: HashMap::new(), + is_sudo, + ci_env: CiEnv::current(), + delayed_failures: RefCell::new(Vec::new()), + prerelease_version: Cell::new(None), + tool_artifacts: Default::default(), + + #[cfg(feature = "build-metrics")] + metrics: metrics::BuildMetrics::init(), + }; + + build.verbose("finding compilers"); + cc_detect::find(&mut build); + // When running `setup`, the profile is about to change, so any requirements we have now may + // be different on the next invocation. Don't check for them until the next time x.py is + // run. This is ok because `setup` never runs any build commands, so it won't fail if commands are missing. + if !matches!(build.config.cmd, Subcommand::Setup { .. }) { + build.verbose("running sanity check"); + sanity::check(&mut build); + } + + // If local-rust is the same major.minor as the current version, then force a + // local-rebuild + let local_version_verbose = + output(Command::new(&build.initial_rustc).arg("--version").arg("--verbose")); + let local_release = local_version_verbose + .lines() + .filter_map(|x| x.strip_prefix("release:")) + .next() + .unwrap() + .trim(); + if local_release.split('.').take(2).eq(version.split('.').take(2)) { + build.verbose(&format!("auto-detected local-rebuild {}", local_release)); + build.local_rebuild = true; + } + + // Make sure we update these before gathering metadata so we don't get an error about missing + // Cargo.toml files. + let rust_submodules = [ + "src/tools/rust-installer", + "src/tools/cargo", + "src/tools/rls", + "src/tools/miri", + "library/backtrace", + "library/stdarch", + ]; + for s in rust_submodules { + build.update_submodule(Path::new(s)); + } + + build.verbose("learning about cargo"); + metadata::build(&mut build); + + build + } + + // modified from `check_submodule` and `update_submodule` in bootstrap.py + /// Given a path to the directory of a submodule, update it. + /// + /// `relative_path` should be relative to the root of the git repository, not an absolute path. + pub(crate) fn update_submodule(&self, relative_path: &Path) { + fn dir_is_empty(dir: &Path) -> bool { + t!(std::fs::read_dir(dir)).next().is_none() + } + + if !self.config.submodules(&self.rust_info) { + return; + } + + let absolute_path = self.config.src.join(relative_path); + + // NOTE: The check for the empty directory is here because when running x.py the first time, + // the submodule won't be checked out. Check it out now so we can build it. + if !channel::GitInfo::new(false, &absolute_path).is_git() && !dir_is_empty(&absolute_path) { + return; + } + + // check_submodule + let checked_out_hash = + output(Command::new("git").args(&["rev-parse", "HEAD"]).current_dir(&absolute_path)); + // update_submodules + let recorded = output( + Command::new("git") + .args(&["ls-tree", "HEAD"]) + .arg(relative_path) + .current_dir(&self.config.src), + ); + let actual_hash = recorded + .split_whitespace() + .nth(2) + .unwrap_or_else(|| panic!("unexpected output `{}`", recorded)); + + // update_submodule + if actual_hash == checked_out_hash.trim_end() { + // already checked out + return; + } + + println!("Updating submodule {}", relative_path.display()); + self.run( + Command::new("git") + .args(&["submodule", "-q", "sync"]) + .arg(relative_path) + .current_dir(&self.config.src), + ); + + // Try passing `--progress` to start, then run git again without if that fails. + let update = |progress: bool| { + let mut git = Command::new("git"); + git.args(&["submodule", "update", "--init", "--recursive", "--depth=1"]); + if progress { + git.arg("--progress"); + } + git.arg(relative_path).current_dir(&self.config.src); + git + }; + // NOTE: doesn't use `try_run` because this shouldn't print an error if it fails. + if !update(true).status().map_or(false, |status| status.success()) { + self.run(&mut update(false)); + } + + self.run(Command::new("git").args(&["reset", "-q", "--hard"]).current_dir(&absolute_path)); + self.run(Command::new("git").args(&["clean", "-qdfx"]).current_dir(absolute_path)); + } + + /// If any submodule has been initialized already, sync it unconditionally. + /// This avoids contributors checking in a submodule change by accident. + pub fn maybe_update_submodules(&self) { + // Avoid running git when there isn't a git checkout. + if !self.config.submodules(&self.rust_info) { + return; + } + let output = output( + self.config + .git() + .args(&["config", "--file"]) + .arg(&self.config.src.join(".gitmodules")) + .args(&["--get-regexp", "path"]), + ); + for line in output.lines() { + // Look for `submodule.$name.path = $path` + // Sample output: `submodule.src/rust-installer.path src/tools/rust-installer` + let submodule = Path::new(line.splitn(2, ' ').nth(1).unwrap()); + // Don't update the submodule unless it's already been cloned. + if channel::GitInfo::new(false, submodule).is_git() { + self.update_submodule(submodule); + } + } + } + + /// Executes the entire build, as configured by the flags and configuration. + pub fn build(&mut self) { + unsafe { + job::setup(self); + } + + self.maybe_update_submodules(); + + if let Subcommand::Format { check, paths } = &self.config.cmd { + return format::format(&builder::Builder::new(&self), *check, &paths); + } + + if let Subcommand::Clean { all } = self.config.cmd { + return clean::clean(self, all); + } + + if let Subcommand::Setup { profile } = &self.config.cmd { + return setup::setup(&self.config, *profile); + } + + { + let builder = builder::Builder::new(&self); + if let Some(path) = builder.paths.get(0) { + if path == Path::new("nonexistent/path/to/trigger/cargo/metadata") { + return; + } + } + } + + if !self.config.dry_run { + { + self.config.dry_run = true; + let builder = builder::Builder::new(&self); + builder.execute_cli(); + } + self.config.dry_run = false; + let builder = builder::Builder::new(&self); + builder.execute_cli(); + } else { + let builder = builder::Builder::new(&self); + builder.execute_cli(); + } + + // Check for postponed failures from `test --no-fail-fast`. + let failures = self.delayed_failures.borrow(); + if failures.len() > 0 { + eprintln!("\n{} command(s) did not execute successfully:\n", failures.len()); + for failure in failures.iter() { + eprintln!(" - {}\n", failure); + } + detail_exit(1); + } + + #[cfg(feature = "build-metrics")] + self.metrics.persist(self); + } + + /// Clear out `dir` if `input` is newer. + /// + /// After this executes, it will also ensure that `dir` exists. + fn clear_if_dirty(&self, dir: &Path, input: &Path) -> bool { + let stamp = dir.join(".stamp"); + let mut cleared = false; + if mtime(&stamp) < mtime(input) { + self.verbose(&format!("Dirty - {}", dir.display())); + let _ = fs::remove_dir_all(dir); + cleared = true; + } else if stamp.exists() { + return cleared; + } + t!(fs::create_dir_all(dir)); + t!(File::create(stamp)); + cleared + } + + /// Gets the space-separated set of activated features for the standard + /// library. + fn std_features(&self, target: TargetSelection) -> String { + let mut features = "panic-unwind".to_string(); + + match self.config.llvm_libunwind(target) { + LlvmLibunwind::InTree => features.push_str(" llvm-libunwind"), + LlvmLibunwind::System => features.push_str(" system-llvm-libunwind"), + LlvmLibunwind::No => {} + } + if self.config.backtrace { + features.push_str(" backtrace"); + } + if self.config.profiler_enabled(target) { + features.push_str(" profiler"); + } + features + } + + /// Gets the space-separated set of activated features for the compiler. + fn rustc_features(&self, kind: Kind) -> String { + let mut features = vec![]; + if self.config.jemalloc { + features.push("jemalloc"); + } + if self.config.llvm_enabled() || kind == Kind::Check { + features.push("llvm"); + } + // keep in sync with `bootstrap/compile.rs:rustc_cargo_env` + if self.config.rustc_parallel { + features.push("rustc_use_parallel_compiler"); + } + + // If debug logging is on, then we want the default for tracing: + // https://github.com/tokio-rs/tracing/blob/3dd5c03d907afdf2c39444a29931833335171554/tracing/src/level_filters.rs#L26 + // which is everything (including debug/trace/etc.) + // if its unset, if debug_assertions is on, then debug_logging will also be on + // as well as tracing *ignoring* this feature when debug_assertions is on + if !self.config.rust_debug_logging { + features.push("max_level_info"); + } + + features.join(" ") + } + + /// Component directory that Cargo will produce output into (e.g. + /// release/debug) + fn cargo_dir(&self) -> &'static str { + if self.config.rust_optimize { "release" } else { "debug" } + } + + fn tools_dir(&self, compiler: Compiler) -> PathBuf { + let out = self + .out + .join(&*compiler.host.triple) + .join(format!("stage{}-tools-bin", compiler.stage)); + t!(fs::create_dir_all(&out)); + out + } + + /// Returns the root directory for all output generated in a particular + /// stage when running with a particular host compiler. + /// + /// The mode indicates what the root directory is for. + fn stage_out(&self, compiler: Compiler, mode: Mode) -> PathBuf { + let suffix = match mode { + Mode::Std => "-std", + Mode::Rustc => "-rustc", + Mode::Codegen => "-codegen", + Mode::ToolBootstrap => "-bootstrap-tools", + Mode::ToolStd | Mode::ToolRustc => "-tools", + }; + self.out.join(&*compiler.host.triple).join(format!("stage{}{}", compiler.stage, suffix)) + } + + /// Returns the root output directory for all Cargo output in a given stage, + /// running a particular compiler, whether or not we're building the + /// standard library, and targeting the specified architecture. + fn cargo_out(&self, compiler: Compiler, mode: Mode, target: TargetSelection) -> PathBuf { + self.stage_out(compiler, mode).join(&*target.triple).join(self.cargo_dir()) + } + + /// Root output directory for LLVM compiled for `target` + /// + /// Note that if LLVM is configured externally then the directory returned + /// will likely be empty. + fn llvm_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("llvm") + } + + fn lld_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("lld") + } + + /// Output directory for all documentation for a target + fn doc_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("doc") + } + + fn test_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("test") + } + + /// Output directory for all documentation for a target + fn compiler_doc_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("compiler-doc") + } + + /// Output directory for some generated md crate documentation for a target (temporary) + fn md_doc_out(&self, target: TargetSelection) -> Interned<PathBuf> { + INTERNER.intern_path(self.out.join(&*target.triple).join("md-doc")) + } + + /// Returns `true` if no custom `llvm-config` is set for the specified target. + /// + /// If no custom `llvm-config` was specified then Rust's llvm will be used. + fn is_rust_llvm(&self, target: TargetSelection) -> bool { + if self.config.llvm_from_ci && target == self.config.build { + return true; + } + + match self.config.target_config.get(&target) { + Some(ref c) => c.llvm_config.is_none(), + None => true, + } + } + + /// Returns the path to `FileCheck` binary for the specified target + fn llvm_filecheck(&self, target: TargetSelection) -> PathBuf { + let target_config = self.config.target_config.get(&target); + if let Some(s) = target_config.and_then(|c| c.llvm_filecheck.as_ref()) { + s.to_path_buf() + } else if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) { + let llvm_bindir = output(Command::new(s).arg("--bindir")); + let filecheck = Path::new(llvm_bindir.trim()).join(exe("FileCheck", target)); + if filecheck.exists() { + filecheck + } else { + // On Fedora the system LLVM installs FileCheck in the + // llvm subdirectory of the libdir. + let llvm_libdir = output(Command::new(s).arg("--libdir")); + let lib_filecheck = + Path::new(llvm_libdir.trim()).join("llvm").join(exe("FileCheck", target)); + if lib_filecheck.exists() { + lib_filecheck + } else { + // Return the most normal file name, even though + // it doesn't exist, so that any error message + // refers to that. + filecheck + } + } + } else { + let base = self.llvm_out(target).join("build"); + let base = if !self.ninja() && target.contains("msvc") { + if self.config.llvm_optimize { + if self.config.llvm_release_debuginfo { + base.join("RelWithDebInfo") + } else { + base.join("Release") + } + } else { + base.join("Debug") + } + } else { + base + }; + base.join("bin").join(exe("FileCheck", target)) + } + } + + /// Directory for libraries built from C/C++ code and shared between stages. + fn native_dir(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("native") + } + + /// Root output directory for rust_test_helpers library compiled for + /// `target` + fn test_helpers_out(&self, target: TargetSelection) -> PathBuf { + self.native_dir(target).join("rust-test-helpers") + } + + /// Adds the `RUST_TEST_THREADS` env var if necessary + fn add_rust_test_threads(&self, cmd: &mut Command) { + if env::var_os("RUST_TEST_THREADS").is_none() { + cmd.env("RUST_TEST_THREADS", self.jobs().to_string()); + } + } + + /// Returns the libdir of the snapshot compiler. + fn rustc_snapshot_libdir(&self) -> PathBuf { + self.rustc_snapshot_sysroot().join(libdir(self.config.build)) + } + + /// Returns the sysroot of the snapshot compiler. + fn rustc_snapshot_sysroot(&self) -> &Path { + static SYSROOT_CACHE: OnceCell<PathBuf> = once_cell::sync::OnceCell::new(); + SYSROOT_CACHE.get_or_init(|| { + let mut rustc = Command::new(&self.initial_rustc); + rustc.args(&["--print", "sysroot"]); + output(&mut rustc).trim().into() + }) + } + + /// Runs a command, printing out nice contextual information if it fails. + fn run(&self, cmd: &mut Command) { + if self.config.dry_run { + return; + } + self.verbose(&format!("running: {:?}", cmd)); + run(cmd, self.is_verbose()) + } + + /// Runs a command, printing out nice contextual information if it fails. + fn run_quiet(&self, cmd: &mut Command) { + if self.config.dry_run { + return; + } + self.verbose(&format!("running: {:?}", cmd)); + run_suppressed(cmd) + } + + /// Runs a command, printing out nice contextual information if it fails. + /// Exits if the command failed to execute at all, otherwise returns its + /// `status.success()`. + fn try_run(&self, cmd: &mut Command) -> bool { + if self.config.dry_run { + return true; + } + self.verbose(&format!("running: {:?}", cmd)); + try_run(cmd, self.is_verbose()) + } + + /// Runs a command, printing out nice contextual information if it fails. + /// Exits if the command failed to execute at all, otherwise returns its + /// `status.success()`. + fn try_run_quiet(&self, cmd: &mut Command) -> bool { + if self.config.dry_run { + return true; + } + self.verbose(&format!("running: {:?}", cmd)); + try_run_suppressed(cmd) + } + + /// Runs a command, printing out nice contextual information if it fails. + /// Returns false if do not execute at all, otherwise returns its + /// `status.success()`. + fn check_run(&self, cmd: &mut Command) -> bool { + if self.config.dry_run { + return true; + } + self.verbose(&format!("running: {:?}", cmd)); + check_run(cmd, self.is_verbose()) + } + + pub fn is_verbose(&self) -> bool { + self.verbosity > 0 + } + + /// Prints a message if this build is configured in verbose mode. + fn verbose(&self, msg: &str) { + if self.is_verbose() { + println!("{}", msg); + } + } + + pub fn is_verbose_than(&self, level: usize) -> bool { + self.verbosity > level + } + + /// Prints a message if this build is configured in more verbose mode than `level`. + fn verbose_than(&self, level: usize, msg: &str) { + if self.is_verbose_than(level) { + println!("{}", msg); + } + } + + fn info(&self, msg: &str) { + if self.config.dry_run { + return; + } + println!("{}", msg); + } + + /// Returns the number of parallel jobs that have been configured for this + /// build. + fn jobs(&self) -> u32 { + self.config.jobs.unwrap_or_else(|| num_cpus::get() as u32) + } + + fn debuginfo_map_to(&self, which: GitRepo) -> Option<String> { + if !self.config.rust_remap_debuginfo { + return None; + } + + match which { + GitRepo::Rustc => { + let sha = self.rust_sha().unwrap_or(&self.version); + Some(format!("/rustc/{}", sha)) + } + GitRepo::Llvm => Some(String::from("/rustc/llvm")), + } + } + + /// Returns the path to the C compiler for the target specified. + fn cc(&self, target: TargetSelection) -> &Path { + self.cc[&target].path() + } + + /// Returns a list of flags to pass to the C compiler for the target + /// specified. + fn cflags(&self, target: TargetSelection, which: GitRepo, c: CLang) -> Vec<String> { + let base = match c { + CLang::C => &self.cc[&target], + CLang::Cxx => &self.cxx[&target], + }; + + // Filter out -O and /O (the optimization flags) that we picked up from + // cc-rs because the build scripts will determine that for themselves. + let mut base = base + .args() + .iter() + .map(|s| s.to_string_lossy().into_owned()) + .filter(|s| !s.starts_with("-O") && !s.starts_with("/O")) + .collect::<Vec<String>>(); + + // If we're compiling on macOS then we add a few unconditional flags + // indicating that we want libc++ (more filled out than libstdc++) and + // we want to compile for 10.7. This way we can ensure that + // LLVM/etc are all properly compiled. + if target.contains("apple-darwin") { + base.push("-stdlib=libc++".into()); + } + + // Work around an apparently bad MinGW / GCC optimization, + // See: https://lists.llvm.org/pipermail/cfe-dev/2016-December/051980.html + // See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=78936 + if &*target.triple == "i686-pc-windows-gnu" { + base.push("-fno-omit-frame-pointer".into()); + } + + if let Some(map_to) = self.debuginfo_map_to(which) { + let map = format!("{}={}", self.src.display(), map_to); + let cc = self.cc(target); + if cc.ends_with("clang") || cc.ends_with("gcc") { + base.push(format!("-fdebug-prefix-map={}", map)); + } else if cc.ends_with("clang-cl.exe") { + base.push("-Xclang".into()); + base.push(format!("-fdebug-prefix-map={}", map)); + } + } + base + } + + /// Returns the path to the `ar` archive utility for the target specified. + fn ar(&self, target: TargetSelection) -> Option<&Path> { + self.ar.get(&target).map(|p| &**p) + } + + /// Returns the path to the `ranlib` utility for the target specified. + fn ranlib(&self, target: TargetSelection) -> Option<&Path> { + self.ranlib.get(&target).map(|p| &**p) + } + + /// Returns the path to the C++ compiler for the target specified. + fn cxx(&self, target: TargetSelection) -> Result<&Path, String> { + match self.cxx.get(&target) { + Some(p) => Ok(p.path()), + None => { + Err(format!("target `{}` is not configured as a host, only as a target", target)) + } + } + } + + /// Returns the path to the linker for the given target if it needs to be overridden. + fn linker(&self, target: TargetSelection) -> Option<&Path> { + if let Some(linker) = self.config.target_config.get(&target).and_then(|c| c.linker.as_ref()) + { + Some(linker) + } else if target.contains("vxworks") { + // need to use CXX compiler as linker to resolve the exception functions + // that are only existed in CXX libraries + Some(self.cxx[&target].path()) + } else if target != self.config.build + && util::use_host_linker(target) + && !target.contains("msvc") + { + Some(self.cc(target)) + } else if self.config.use_lld && !self.is_fuse_ld_lld(target) && self.build == target { + Some(&self.initial_lld) + } else { + None + } + } + + // LLD is used through `-fuse-ld=lld` rather than directly. + // Only MSVC targets use LLD directly at the moment. + fn is_fuse_ld_lld(&self, target: TargetSelection) -> bool { + self.config.use_lld && !target.contains("msvc") + } + + fn lld_flags(&self, target: TargetSelection) -> impl Iterator<Item = String> { + let mut options = [None, None]; + + if self.config.use_lld { + if self.is_fuse_ld_lld(target) { + options[0] = Some("-Clink-arg=-fuse-ld=lld".to_string()); + } + + let threads = if target.contains("windows") { "/threads:1" } else { "--threads=1" }; + options[1] = Some(format!("-Clink-arg=-Wl,{}", threads)); + } + + IntoIterator::into_iter(options).flatten() + } + + /// Returns if this target should statically link the C runtime, if specified + fn crt_static(&self, target: TargetSelection) -> Option<bool> { + if target.contains("pc-windows-msvc") { + Some(true) + } else { + self.config.target_config.get(&target).and_then(|t| t.crt_static) + } + } + + /// Returns the "musl root" for this `target`, if defined + fn musl_root(&self, target: TargetSelection) -> Option<&Path> { + self.config + .target_config + .get(&target) + .and_then(|t| t.musl_root.as_ref()) + .or_else(|| self.config.musl_root.as_ref()) + .map(|p| &**p) + } + + /// Returns the "musl libdir" for this `target`. + fn musl_libdir(&self, target: TargetSelection) -> Option<PathBuf> { + let t = self.config.target_config.get(&target)?; + if let libdir @ Some(_) = &t.musl_libdir { + return libdir.clone(); + } + self.musl_root(target).map(|root| root.join("lib")) + } + + /// Returns the sysroot for the wasi target, if defined + fn wasi_root(&self, target: TargetSelection) -> Option<&Path> { + self.config.target_config.get(&target).and_then(|t| t.wasi_root.as_ref()).map(|p| &**p) + } + + /// Returns `true` if this is a no-std `target`, if defined + fn no_std(&self, target: TargetSelection) -> Option<bool> { + self.config.target_config.get(&target).map(|t| t.no_std) + } + + /// Returns `true` if the target will be tested using the `remote-test-client` + /// and `remote-test-server` binaries. + fn remote_tested(&self, target: TargetSelection) -> bool { + self.qemu_rootfs(target).is_some() + || target.contains("android") + || env::var_os("TEST_DEVICE_ADDR").is_some() + } + + /// Returns the root of the "rootfs" image that this target will be using, + /// if one was configured. + /// + /// If `Some` is returned then that means that tests for this target are + /// emulated with QEMU and binaries will need to be shipped to the emulator. + fn qemu_rootfs(&self, target: TargetSelection) -> Option<&Path> { + self.config.target_config.get(&target).and_then(|t| t.qemu_rootfs.as_ref()).map(|p| &**p) + } + + /// Path to the python interpreter to use + fn python(&self) -> &Path { + if self.config.build.ends_with("apple-darwin") { + // Force /usr/bin/python3 on macOS for LLDB tests because we're loading the + // LLDB plugin's compiled module which only works with the system python + // (namely not Homebrew-installed python) + Path::new("/usr/bin/python3") + } else { + self.config + .python + .as_ref() + .expect("python is required for running LLDB or rustdoc tests") + } + } + + /// Temporary directory that extended error information is emitted to. + fn extended_error_dir(&self) -> PathBuf { + self.out.join("tmp/extended-error-metadata") + } + + /// Tests whether the `compiler` compiling for `target` should be forced to + /// use a stage1 compiler instead. + /// + /// Currently, by default, the build system does not perform a "full + /// bootstrap" by default where we compile the compiler three times. + /// Instead, we compile the compiler two times. The final stage (stage2) + /// just copies the libraries from the previous stage, which is what this + /// method detects. + /// + /// Here we return `true` if: + /// + /// * The build isn't performing a full bootstrap + /// * The `compiler` is in the final stage, 2 + /// * We're not cross-compiling, so the artifacts are already available in + /// stage1 + /// + /// When all of these conditions are met the build will lift artifacts from + /// the previous stage forward. + fn force_use_stage1(&self, compiler: Compiler, target: TargetSelection) -> bool { + !self.config.full_bootstrap + && compiler.stage >= 2 + && (self.hosts.iter().any(|h| *h == target) || target == self.build) + } + + /// Given `num` in the form "a.b.c" return a "release string" which + /// describes the release version number. + /// + /// For example on nightly this returns "a.b.c-nightly", on beta it returns + /// "a.b.c-beta.1" and on stable it just returns "a.b.c". + fn release(&self, num: &str) -> String { + match &self.config.channel[..] { + "stable" => num.to_string(), + "beta" => { + if self.rust_info.is_git() && !self.config.ignore_git { + format!("{}-beta.{}", num, self.beta_prerelease_version()) + } else { + format!("{}-beta", num) + } + } + "nightly" => format!("{}-nightly", num), + _ => format!("{}-dev", num), + } + } + + fn beta_prerelease_version(&self) -> u32 { + if let Some(s) = self.prerelease_version.get() { + return s; + } + + // Figure out how many merge commits happened since we branched off master. + // That's our beta number! + // (Note that we use a `..` range, not the `...` symmetric difference.) + let count = + output(self.config.git().arg("rev-list").arg("--count").arg("--merges").arg(format!( + "refs/remotes/origin/{}..HEAD", + self.config.stage0_metadata.config.nightly_branch + ))); + let n = count.trim().parse().unwrap(); + self.prerelease_version.set(Some(n)); + n + } + + /// Returns the value of `release` above for Rust itself. + fn rust_release(&self) -> String { + self.release(&self.version) + } + + /// Returns the "package version" for a component given the `num` release + /// number. + /// + /// The package version is typically what shows up in the names of tarballs. + /// For channels like beta/nightly it's just the channel name, otherwise + /// it's the `num` provided. + fn package_vers(&self, num: &str) -> String { + match &self.config.channel[..] { + "stable" => num.to_string(), + "beta" => "beta".to_string(), + "nightly" => "nightly".to_string(), + _ => format!("{}-dev", num), + } + } + + /// Returns the value of `package_vers` above for Rust itself. + fn rust_package_vers(&self) -> String { + self.package_vers(&self.version) + } + + fn llvm_link_tools_dynamically(&self, target: TargetSelection) -> bool { + target.contains("linux-gnu") || target.contains("apple-darwin") + } + + /// Returns the `version` string associated with this compiler for Rust + /// itself. + /// + /// Note that this is a descriptive string which includes the commit date, + /// sha, version, etc. + fn rust_version(&self) -> String { + let mut version = self.rust_info.version(self, &self.version); + if let Some(ref s) = self.config.description { + version.push_str(" ("); + version.push_str(s); + version.push(')'); + } + version + } + + /// Returns the full commit hash. + fn rust_sha(&self) -> Option<&str> { + self.rust_info.sha() + } + + /// Returns the `a.b.c` version that the given package is at. + fn release_num(&self, package: &str) -> String { + let toml_file_name = self.src.join(&format!("src/tools/{}/Cargo.toml", package)); + let toml = t!(fs::read_to_string(&toml_file_name)); + for line in toml.lines() { + if let Some(stripped) = + line.strip_prefix("version = \"").and_then(|s| s.strip_suffix("\"")) + { + return stripped.to_owned(); + } + } + + panic!("failed to find version in {}'s Cargo.toml", package) + } + + /// Returns `true` if unstable features should be enabled for the compiler + /// we're building. + fn unstable_features(&self) -> bool { + match &self.config.channel[..] { + "stable" | "beta" => false, + "nightly" | _ => true, + } + } + + /// Returns a Vec of all the dependencies of the given root crate, + /// including transitive dependencies and the root itself. Only includes + /// "local" crates (those in the local source tree, not from a registry). + fn in_tree_crates(&self, root: &str, target: Option<TargetSelection>) -> Vec<&Crate> { + let mut ret = Vec::new(); + let mut list = vec![INTERNER.intern_str(root)]; + let mut visited = HashSet::new(); + while let Some(krate) = list.pop() { + let krate = &self.crates[&krate]; + ret.push(krate); + for dep in &krate.deps { + if !self.crates.contains_key(dep) { + // Ignore non-workspace members. + continue; + } + // Don't include optional deps if their features are not + // enabled. Ideally this would be computed from `cargo + // metadata --features …`, but that is somewhat slow. In + // the future, we may want to consider just filtering all + // build and dev dependencies in metadata::build. + if visited.insert(dep) + && (dep != "profiler_builtins" + || target + .map(|t| self.config.profiler_enabled(t)) + .unwrap_or_else(|| self.config.any_profiler_enabled())) + && (dep != "rustc_codegen_llvm" || self.config.llvm_enabled()) + { + list.push(*dep); + } + } + } + ret + } + + fn read_stamp_file(&self, stamp: &Path) -> Vec<(PathBuf, DependencyType)> { + if self.config.dry_run { + return Vec::new(); + } + + let mut paths = Vec::new(); + let contents = t!(fs::read(stamp), &stamp); + // This is the method we use for extracting paths from the stamp file passed to us. See + // run_cargo for more information (in compile.rs). + for part in contents.split(|b| *b == 0) { + if part.is_empty() { + continue; + } + let dependency_type = match part[0] as char { + 'h' => DependencyType::Host, + 's' => DependencyType::TargetSelfContained, + 't' => DependencyType::Target, + _ => unreachable!(), + }; + let path = PathBuf::from(t!(str::from_utf8(&part[1..]))); + paths.push((path, dependency_type)); + } + paths + } + + /// Create a temporary directory in `out` and return its path. + /// + /// NOTE: this temporary directory is shared between all steps; + /// if you need an empty directory, create a new subdirectory inside it. + fn tempdir(&self) -> PathBuf { + let tmp = self.out.join("tmp"); + t!(fs::create_dir_all(&tmp)); + tmp + } + + /// Copies a file from `src` to `dst` + pub fn copy(&self, src: &Path, dst: &Path) { + self.copy_internal(src, dst, false); + } + + fn copy_internal(&self, src: &Path, dst: &Path, dereference_symlinks: bool) { + if self.config.dry_run { + return; + } + self.verbose_than(1, &format!("Copy {:?} to {:?}", src, dst)); + if src == dst { + return; + } + let _ = fs::remove_file(&dst); + let metadata = t!(src.symlink_metadata()); + let mut src = src.to_path_buf(); + if metadata.file_type().is_symlink() { + if dereference_symlinks { + src = t!(fs::canonicalize(src)); + } else { + let link = t!(fs::read_link(src)); + t!(self.symlink_file(link, dst)); + return; + } + } + if let Ok(()) = fs::hard_link(&src, dst) { + // Attempt to "easy copy" by creating a hard link + // (symlinks don't work on windows), but if that fails + // just fall back to a slow `copy` operation. + } else { + if let Err(e) = fs::copy(&src, dst) { + panic!("failed to copy `{}` to `{}`: {}", src.display(), dst.display(), e) + } + t!(fs::set_permissions(dst, metadata.permissions())); + let atime = FileTime::from_last_access_time(&metadata); + let mtime = FileTime::from_last_modification_time(&metadata); + t!(filetime::set_file_times(dst, atime, mtime)); + } + } + + /// Copies the `src` directory recursively to `dst`. Both are assumed to exist + /// when this function is called. + pub fn cp_r(&self, src: &Path, dst: &Path) { + if self.config.dry_run { + return; + } + for f in self.read_dir(src) { + let path = f.path(); + let name = path.file_name().unwrap(); + let dst = dst.join(name); + if t!(f.file_type()).is_dir() { + t!(fs::create_dir_all(&dst)); + self.cp_r(&path, &dst); + } else { + let _ = fs::remove_file(&dst); + self.copy(&path, &dst); + } + } + } + + /// Copies the `src` directory recursively to `dst`. Both are assumed to exist + /// when this function is called. Unwanted files or directories can be skipped + /// by returning `false` from the filter function. + pub fn cp_filtered(&self, src: &Path, dst: &Path, filter: &dyn Fn(&Path) -> bool) { + // Immediately recurse with an empty relative path + self.recurse_(src, dst, Path::new(""), filter) + } + + // Inner function does the actual work + fn recurse_(&self, src: &Path, dst: &Path, relative: &Path, filter: &dyn Fn(&Path) -> bool) { + for f in self.read_dir(src) { + let path = f.path(); + let name = path.file_name().unwrap(); + let dst = dst.join(name); + let relative = relative.join(name); + // Only copy file or directory if the filter function returns true + if filter(&relative) { + if t!(f.file_type()).is_dir() { + let _ = fs::remove_dir_all(&dst); + self.create_dir(&dst); + self.recurse_(&path, &dst, &relative, filter); + } else { + let _ = fs::remove_file(&dst); + self.copy(&path, &dst); + } + } + } + } + + fn copy_to_folder(&self, src: &Path, dest_folder: &Path) { + let file_name = src.file_name().unwrap(); + let dest = dest_folder.join(file_name); + self.copy(src, &dest); + } + + fn install(&self, src: &Path, dstdir: &Path, perms: u32) { + if self.config.dry_run { + return; + } + let dst = dstdir.join(src.file_name().unwrap()); + self.verbose_than(1, &format!("Install {:?} to {:?}", src, dst)); + t!(fs::create_dir_all(dstdir)); + if !src.exists() { + panic!("Error: File \"{}\" not found!", src.display()); + } + self.copy_internal(src, &dst, true); + chmod(&dst, perms); + } + + fn create(&self, path: &Path, s: &str) { + if self.config.dry_run { + return; + } + t!(fs::write(path, s)); + } + + fn read(&self, path: &Path) -> String { + if self.config.dry_run { + return String::new(); + } + t!(fs::read_to_string(path)) + } + + fn create_dir(&self, dir: &Path) { + if self.config.dry_run { + return; + } + t!(fs::create_dir_all(dir)) + } + + fn remove_dir(&self, dir: &Path) { + if self.config.dry_run { + return; + } + t!(fs::remove_dir_all(dir)) + } + + fn read_dir(&self, dir: &Path) -> impl Iterator<Item = fs::DirEntry> { + let iter = match fs::read_dir(dir) { + Ok(v) => v, + Err(_) if self.config.dry_run => return vec![].into_iter(), + Err(err) => panic!("could not read dir {:?}: {:?}", dir, err), + }; + iter.map(|e| t!(e)).collect::<Vec<_>>().into_iter() + } + + fn symlink_file<P: AsRef<Path>, Q: AsRef<Path>>(&self, src: P, link: Q) -> io::Result<()> { + #[cfg(unix)] + use std::os::unix::fs::symlink as symlink_file; + #[cfg(windows)] + use std::os::windows::fs::symlink_file; + if !self.config.dry_run { symlink_file(src.as_ref(), link.as_ref()) } else { Ok(()) } + } + + fn remove(&self, f: &Path) { + if self.config.dry_run { + return; + } + fs::remove_file(f).unwrap_or_else(|_| panic!("failed to remove {:?}", f)); + } + + /// Returns if config.ninja is enabled, and checks for ninja existence, + /// exiting with a nicer error message if not. + fn ninja(&self) -> bool { + let mut cmd_finder = crate::sanity::Finder::new(); + + if self.config.ninja_in_file { + // Some Linux distros rename `ninja` to `ninja-build`. + // CMake can work with either binary name. + if cmd_finder.maybe_have("ninja-build").is_none() + && cmd_finder.maybe_have("ninja").is_none() + { + eprintln!( + " +Couldn't find required command: ninja (or ninja-build) + +You should install ninja as described at +<https://github.com/ninja-build/ninja/wiki/Pre-built-Ninja-packages>, +or set `ninja = false` in the `[llvm]` section of `config.toml`. +Alternatively, set `download-ci-llvm = true` in that `[llvm]` section +to download LLVM rather than building it. +" + ); + detail_exit(1); + } + } + + // If ninja isn't enabled but we're building for MSVC then we try + // doubly hard to enable it. It was realized in #43767 that the msbuild + // CMake generator for MSVC doesn't respect configuration options like + // disabling LLVM assertions, which can often be quite important! + // + // In these cases we automatically enable Ninja if we find it in the + // environment. + if !self.config.ninja_in_file && self.config.build.contains("msvc") { + if cmd_finder.maybe_have("ninja").is_some() { + return true; + } + } + + self.config.ninja_in_file + } +} + +#[cfg(unix)] +fn chmod(path: &Path, perms: u32) { + use std::os::unix::fs::*; + t!(fs::set_permissions(path, fs::Permissions::from_mode(perms))); +} +#[cfg(windows)] +fn chmod(_path: &Path, _perms: u32) {} + +/// If code is not 0 (successful exit status), exit status is 101 (rust's default error code.) +/// If the test is running and code is an error code, it will cause a panic. +fn detail_exit(code: i32) -> ! { + // Successful exit + if code == 0 { + std::process::exit(0); + } + if cfg!(test) { + panic!("status code: {}", code); + } else { + std::panic::resume_unwind(Box::new(code)); + } +} + +impl Compiler { + pub fn with_stage(mut self, stage: u32) -> Compiler { + self.stage = stage; + self + } + + /// Returns `true` if this is a snapshot compiler for `build`'s configuration + pub fn is_snapshot(&self, build: &Build) -> bool { + self.stage == 0 && self.host == build.build + } + + /// Returns if this compiler should be treated as a final stage one in the + /// current build session. + /// This takes into account whether we're performing a full bootstrap or + /// not; don't directly compare the stage with `2`! + pub fn is_final_stage(&self, build: &Build) -> bool { + let final_stage = if build.config.full_bootstrap { 2 } else { 1 }; + self.stage >= final_stage + } +} + +fn envify(s: &str) -> String { + s.chars() + .map(|c| match c { + '-' => '_', + c => c, + }) + .flat_map(|c| c.to_uppercase()) + .collect() +} diff --git a/src/bootstrap/metadata.rs b/src/bootstrap/metadata.rs new file mode 100644 index 000000000..e193e70a0 --- /dev/null +++ b/src/bootstrap/metadata.rs @@ -0,0 +1,59 @@ +use std::path::PathBuf; +use std::process::Command; + +use serde::Deserialize; + +use crate::cache::INTERNER; +use crate::util::output; +use crate::{Build, Crate}; + +#[derive(Deserialize)] +struct Output { + packages: Vec<Package>, +} + +#[derive(Deserialize)] +struct Package { + name: String, + source: Option<String>, + manifest_path: String, + dependencies: Vec<Dependency>, +} + +#[derive(Deserialize)] +struct Dependency { + name: String, + source: Option<String>, +} + +pub fn build(build: &mut Build) { + // Run `cargo metadata` to figure out what crates we're testing. + let mut cargo = Command::new(&build.initial_cargo); + cargo + .arg("metadata") + .arg("--format-version") + .arg("1") + .arg("--no-deps") + .arg("--manifest-path") + .arg(build.src.join("Cargo.toml")); + let output = output(&mut cargo); + let output: Output = serde_json::from_str(&output).unwrap(); + for package in output.packages { + if package.source.is_none() { + let name = INTERNER.intern_string(package.name); + let mut path = PathBuf::from(package.manifest_path); + path.pop(); + let deps = package + .dependencies + .into_iter() + .filter(|dep| dep.source.is_none()) + .map(|dep| INTERNER.intern_string(dep.name)) + .collect(); + let krate = Crate { name, deps, path }; + let relative_path = krate.local_path(build); + build.crates.insert(name, krate); + let existing_path = build.crate_paths.insert(relative_path, name); + assert!(existing_path.is_none(), "multiple crates with the same path"); + } + } +} diff --git a/src/bootstrap/metrics.rs b/src/bootstrap/metrics.rs new file mode 100644 index 000000000..451febddc --- /dev/null +++ b/src/bootstrap/metrics.rs @@ -0,0 +1,208 @@ +//! This module is responsible for collecting metrics profiling information for the current build +//! and dumping it to disk as JSON, to aid investigations on build and CI performance. +//! +//! As this module requires additional dependencies not present during local builds, it's cfg'd +//! away whenever the `build.metrics` config option is not set to `true`. + +use crate::builder::Step; +use crate::util::t; +use crate::Build; +use serde::{Deserialize, Serialize}; +use std::cell::RefCell; +use std::fs::File; +use std::io::BufWriter; +use std::time::{Duration, Instant}; +use sysinfo::{CpuExt, System, SystemExt}; + +pub(crate) struct BuildMetrics { + state: RefCell<MetricsState>, +} + +impl BuildMetrics { + pub(crate) fn init() -> Self { + let state = RefCell::new(MetricsState { + finished_steps: Vec::new(), + running_steps: Vec::new(), + + system_info: System::new(), + timer_start: None, + invocation_timer_start: Instant::now(), + }); + + BuildMetrics { state } + } + + pub(crate) fn enter_step<S: Step>(&self, step: &S) { + let mut state = self.state.borrow_mut(); + + // Consider all the stats gathered so far as the parent's. + if !state.running_steps.is_empty() { + self.collect_stats(&mut *state); + } + + state.system_info.refresh_cpu(); + state.timer_start = Some(Instant::now()); + + state.running_steps.push(StepMetrics { + type_: std::any::type_name::<S>().into(), + debug_repr: format!("{step:?}"), + + cpu_usage_time_sec: 0.0, + duration_excluding_children_sec: Duration::ZERO, + + children: Vec::new(), + }); + } + + pub(crate) fn exit_step(&self) { + let mut state = self.state.borrow_mut(); + + self.collect_stats(&mut *state); + + let step = state.running_steps.pop().unwrap(); + if state.running_steps.is_empty() { + state.finished_steps.push(step); + state.timer_start = None; + } else { + state.running_steps.last_mut().unwrap().children.push(step); + + // Start collecting again for the parent step. + state.system_info.refresh_cpu(); + state.timer_start = Some(Instant::now()); + } + } + + fn collect_stats(&self, state: &mut MetricsState) { + let step = state.running_steps.last_mut().unwrap(); + + let elapsed = state.timer_start.unwrap().elapsed(); + step.duration_excluding_children_sec += elapsed; + + state.system_info.refresh_cpu(); + let cpu = state.system_info.cpus().iter().map(|p| p.cpu_usage()).sum::<f32>(); + step.cpu_usage_time_sec += cpu as f64 / 100.0 * elapsed.as_secs_f64(); + } + + pub(crate) fn persist(&self, build: &Build) { + let mut state = self.state.borrow_mut(); + assert!(state.running_steps.is_empty(), "steps are still executing"); + + let dest = build.out.join("metrics.json"); + + let mut system = System::new(); + system.refresh_cpu(); + system.refresh_memory(); + + let system_stats = JsonInvocationSystemStats { + cpu_threads_count: system.cpus().len(), + cpu_model: system.cpus()[0].brand().into(), + + memory_total_bytes: system.total_memory() * 1024, + }; + let steps = std::mem::take(&mut state.finished_steps); + + // Some of our CI builds consist of multiple independent CI invocations. Ensure all the + // previous invocations are still present in the resulting file. + let mut invocations = match std::fs::read(&dest) { + Ok(contents) => t!(serde_json::from_slice::<JsonRoot>(&contents)).invocations, + Err(err) => { + if err.kind() != std::io::ErrorKind::NotFound { + panic!("failed to open existing metrics file at {}: {err}", dest.display()); + } + Vec::new() + } + }; + invocations.push(JsonInvocation { + duration_including_children_sec: state.invocation_timer_start.elapsed().as_secs_f64(), + children: steps.into_iter().map(|step| self.prepare_json_step(step)).collect(), + }); + + let json = JsonRoot { system_stats, invocations }; + + t!(std::fs::create_dir_all(dest.parent().unwrap())); + let mut file = BufWriter::new(t!(File::create(&dest))); + t!(serde_json::to_writer(&mut file, &json)); + } + + fn prepare_json_step(&self, step: StepMetrics) -> JsonNode { + JsonNode::RustbuildStep { + type_: step.type_, + debug_repr: step.debug_repr, + + duration_excluding_children_sec: step.duration_excluding_children_sec.as_secs_f64(), + system_stats: JsonStepSystemStats { + cpu_utilization_percent: step.cpu_usage_time_sec * 100.0 + / step.duration_excluding_children_sec.as_secs_f64(), + }, + + children: step + .children + .into_iter() + .map(|child| self.prepare_json_step(child)) + .collect(), + } + } +} + +struct MetricsState { + finished_steps: Vec<StepMetrics>, + running_steps: Vec<StepMetrics>, + + system_info: System, + timer_start: Option<Instant>, + invocation_timer_start: Instant, +} + +struct StepMetrics { + type_: String, + debug_repr: String, + + cpu_usage_time_sec: f64, + duration_excluding_children_sec: Duration, + + children: Vec<StepMetrics>, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +struct JsonRoot { + system_stats: JsonInvocationSystemStats, + invocations: Vec<JsonInvocation>, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +struct JsonInvocation { + duration_including_children_sec: f64, + children: Vec<JsonNode>, +} + +#[derive(Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum JsonNode { + RustbuildStep { + #[serde(rename = "type")] + type_: String, + debug_repr: String, + + duration_excluding_children_sec: f64, + system_stats: JsonStepSystemStats, + + children: Vec<JsonNode>, + }, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +struct JsonInvocationSystemStats { + cpu_threads_count: usize, + cpu_model: String, + + memory_total_bytes: u64, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +struct JsonStepSystemStats { + cpu_utilization_percent: f64, +} diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in new file mode 100644 index 000000000..5a1f2e704 --- /dev/null +++ b/src/bootstrap/mk/Makefile.in @@ -0,0 +1,83 @@ +ifdef VERBOSE +Q := +BOOTSTRAP_ARGS := -v +else +Q := @ +BOOTSTRAP_ARGS := +endif + +BOOTSTRAP := $(CFG_PYTHON) $(CFG_SRC_DIR)src/bootstrap/bootstrap.py + +all: + $(Q)$(BOOTSTRAP) build --stage 2 $(BOOTSTRAP_ARGS) + $(Q)$(BOOTSTRAP) doc --stage 2 $(BOOTSTRAP_ARGS) + +help: + $(Q)echo 'Welcome to the rustbuild build system!' + $(Q)echo + $(Q)echo This makefile is a thin veneer over the ./x.py script located + $(Q)echo in this directory. To get the full power of the build system + $(Q)echo you can run x.py directly. + $(Q)echo + $(Q)echo To learn more run \`./x.py --help\` + +clean: + $(Q)$(BOOTSTRAP) clean $(BOOTSTRAP_ARGS) + +rustc-stage1: + $(Q)$(BOOTSTRAP) build --stage 1 library/test $(BOOTSTRAP_ARGS) +rustc-stage2: + $(Q)$(BOOTSTRAP) build --stage 2 library/test $(BOOTSTRAP_ARGS) + +docs: doc +doc: + $(Q)$(BOOTSTRAP) doc --stage 2 $(BOOTSTRAP_ARGS) +nomicon: + $(Q)$(BOOTSTRAP) doc --stage 2 src/doc/nomicon $(BOOTSTRAP_ARGS) +book: + $(Q)$(BOOTSTRAP) doc --stage 2 src/doc/book $(BOOTSTRAP_ARGS) +standalone-docs: + $(Q)$(BOOTSTRAP) doc --stage 2 src/doc $(BOOTSTRAP_ARGS) +check: + $(Q)$(BOOTSTRAP) test --stage 2 $(BOOTSTRAP_ARGS) +check-aux: + $(Q)$(BOOTSTRAP) test --stage 2 \ + src/tools/cargo \ + src/tools/cargotest \ + $(BOOTSTRAP_ARGS) +dist: + $(Q)$(BOOTSTRAP) dist $(BOOTSTRAP_ARGS) +distcheck: + $(Q)$(BOOTSTRAP) dist $(BOOTSTRAP_ARGS) + $(Q)$(BOOTSTRAP) test --stage 2 distcheck $(BOOTSTRAP_ARGS) +install: + $(Q)$(BOOTSTRAP) install $(BOOTSTRAP_ARGS) +tidy: + $(Q)$(BOOTSTRAP) test --stage 2 src/tools/tidy $(BOOTSTRAP_ARGS) +prepare: + $(Q)$(BOOTSTRAP) build --stage 2 nonexistent/path/to/trigger/cargo/metadata + +check-stage2-T-arm-linux-androideabi-H-x86_64-unknown-linux-gnu: + $(Q)$(BOOTSTRAP) test --stage 2 --target arm-linux-androideabi +check-stage2-T-x86_64-unknown-linux-musl-H-x86_64-unknown-linux-gnu: + $(Q)$(BOOTSTRAP) test --stage 2 --target x86_64-unknown-linux-musl + +TESTS_IN_2 := \ + src/test/ui \ + src/tools/linkchecker + +ci-subset-1: + $(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_2:%=--exclude %) +ci-subset-2: + $(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_2) + +TESTS_IN_MINGW_2 := \ + src/test/ui + +ci-mingw-subset-1: + $(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_MINGW_2:%=--exclude %) +ci-mingw-subset-2: + $(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_MINGW_2) + + +.PHONY: dist diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs new file mode 100644 index 000000000..4d548dbb6 --- /dev/null +++ b/src/bootstrap/native.rs @@ -0,0 +1,1335 @@ +//! Compilation of native dependencies like LLVM. +//! +//! Native projects like LLVM unfortunately aren't suited just yet for +//! compilation in build scripts that Cargo has. This is because the +//! compilation takes a *very* long time but also because we don't want to +//! compile LLVM 3 times as part of a normal bootstrap (we want it cached). +//! +//! LLVM and compiler-rt are essentially just wired up to everything else to +//! ensure that they're always in place if needed. + +use std::env; +use std::env::consts::EXE_EXTENSION; +use std::ffi::{OsStr, OsString}; +use std::fs::{self, File}; +use std::io; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use crate::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::config::TargetSelection; +use crate::util::get_clang_cl_resource_dir; +use crate::util::{self, exe, output, program_out_of_date, t, up_to_date}; +use crate::{CLang, GitRepo}; + +pub struct Meta { + stamp: HashStamp, + build_llvm_config: PathBuf, + out_dir: PathBuf, + root: String, +} + +// Linker flags to pass to LLVM's CMake invocation. +#[derive(Debug, Clone, Default)] +struct LdFlags { + // CMAKE_EXE_LINKER_FLAGS + exe: OsString, + // CMAKE_SHARED_LINKER_FLAGS + shared: OsString, + // CMAKE_MODULE_LINKER_FLAGS + module: OsString, +} + +impl LdFlags { + fn push_all(&mut self, s: impl AsRef<OsStr>) { + let s = s.as_ref(); + self.exe.push(" "); + self.exe.push(s); + self.shared.push(" "); + self.shared.push(s); + self.module.push(" "); + self.module.push(s); + } +} + +// This returns whether we've already previously built LLVM. +// +// It's used to avoid busting caches during x.py check -- if we've already built +// LLVM, it's fine for us to not try to avoid doing so. +// +// This will return the llvm-config if it can get it (but it will not build it +// if not). +pub fn prebuilt_llvm_config( + builder: &Builder<'_>, + target: TargetSelection, +) -> Result<PathBuf, Meta> { + maybe_download_ci_llvm(builder); + + // If we're using a custom LLVM bail out here, but we can only use a + // custom LLVM for the build triple. + if let Some(config) = builder.config.target_config.get(&target) { + if let Some(ref s) = config.llvm_config { + check_llvm_version(builder, s); + return Ok(s.to_path_buf()); + } + } + + let root = "src/llvm-project/llvm"; + let out_dir = builder.llvm_out(target); + + let mut llvm_config_ret_dir = builder.llvm_out(builder.config.build); + if !builder.config.build.contains("msvc") || builder.ninja() { + llvm_config_ret_dir.push("build"); + } + llvm_config_ret_dir.push("bin"); + + let build_llvm_config = llvm_config_ret_dir.join(exe("llvm-config", builder.config.build)); + + let stamp = out_dir.join("llvm-finished-building"); + let stamp = HashStamp::new(stamp, builder.in_tree_llvm_info.sha()); + + if builder.config.llvm_skip_rebuild && stamp.path.exists() { + builder.info( + "Warning: \ + Using a potentially stale build of LLVM; \ + This may not behave well.", + ); + return Ok(build_llvm_config); + } + + if stamp.is_done() { + if stamp.hash.is_none() { + builder.info( + "Could not determine the LLVM submodule commit hash. \ + Assuming that an LLVM rebuild is not necessary.", + ); + builder.info(&format!( + "To force LLVM to rebuild, remove the file `{}`", + stamp.path.display() + )); + } + return Ok(build_llvm_config); + } + + Err(Meta { stamp, build_llvm_config, out_dir, root: root.into() }) +} + +pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) { + let config = &builder.config; + if !config.llvm_from_ci { + return; + } + let mut rev_list = config.git(); + rev_list.args(&[ + PathBuf::from("rev-list"), + format!("--author={}", builder.config.stage0_metadata.config.git_merge_commit_email).into(), + "-n1".into(), + "--first-parent".into(), + "HEAD".into(), + "--".into(), + builder.src.join("src/llvm-project"), + builder.src.join("src/bootstrap/download-ci-llvm-stamp"), + // the LLVM shared object file is named `LLVM-12-rust-{version}-nightly` + builder.src.join("src/version"), + ]); + let llvm_sha = output(&mut rev_list); + let llvm_sha = llvm_sha.trim(); + + if llvm_sha == "" { + eprintln!("error: could not find commit hash for downloading LLVM"); + eprintln!("help: maybe your repository history is too shallow?"); + eprintln!("help: consider disabling `download-ci-llvm`"); + eprintln!("help: or fetch enough history to include one upstream commit"); + panic!(); + } + + let llvm_root = config.ci_llvm_root(); + let llvm_stamp = llvm_root.join(".llvm-stamp"); + let key = format!("{}{}", llvm_sha, config.llvm_assertions); + if program_out_of_date(&llvm_stamp, &key) && !config.dry_run { + download_ci_llvm(builder, &llvm_sha); + for entry in t!(fs::read_dir(llvm_root.join("bin"))) { + builder.fix_bin_or_dylib(&t!(entry).path()); + } + + // Update the timestamp of llvm-config to force rustc_llvm to be + // rebuilt. This is a hacky workaround for a deficiency in Cargo where + // the rerun-if-changed directive doesn't handle changes very well. + // https://github.com/rust-lang/cargo/issues/10791 + // Cargo only compares the timestamp of the file relative to the last + // time `rustc_llvm` build script ran. However, the timestamps of the + // files in the tarball are in the past, so it doesn't trigger a + // rebuild. + let now = filetime::FileTime::from_system_time(std::time::SystemTime::now()); + let llvm_config = llvm_root.join("bin").join(exe("llvm-config", builder.config.build)); + t!(filetime::set_file_times(&llvm_config, now, now)); + + let llvm_lib = llvm_root.join("lib"); + for entry in t!(fs::read_dir(&llvm_lib)) { + let lib = t!(entry).path(); + if lib.extension().map_or(false, |ext| ext == "so") { + builder.fix_bin_or_dylib(&lib); + } + } + t!(fs::write(llvm_stamp, key)); + } +} + +fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) { + let llvm_assertions = builder.config.llvm_assertions; + + let cache_prefix = format!("llvm-{}-{}", llvm_sha, llvm_assertions); + let cache_dst = builder.out.join("cache"); + let rustc_cache = cache_dst.join(cache_prefix); + if !rustc_cache.exists() { + t!(fs::create_dir_all(&rustc_cache)); + } + let base = if llvm_assertions { + &builder.config.stage0_metadata.config.artifacts_with_llvm_assertions_server + } else { + &builder.config.stage0_metadata.config.artifacts_server + }; + let channel = builder.config.artifact_channel(llvm_sha); + let filename = format!("rust-dev-{}-{}.tar.xz", channel, builder.build.build.triple); + let tarball = rustc_cache.join(&filename); + if !tarball.exists() { + let help_on_error = "error: failed to download llvm from ci + +help: old builds get deleted after a certain time +help: if trying to compile an old commit of rustc, disable `download-ci-llvm` in config.toml: + +[llvm] +download-ci-llvm = false +"; + builder.download_component( + &format!("{base}/{llvm_sha}/{filename}"), + &tarball, + help_on_error, + ); + } + let llvm_root = builder.config.ci_llvm_root(); + builder.unpack(&tarball, &llvm_root, "rust-dev"); +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Llvm { + pub target: TargetSelection, +} + +impl Step for Llvm { + type Output = PathBuf; // path to llvm-config + + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/llvm-project").path("src/llvm-project/llvm") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Llvm { target: run.target }); + } + + /// Compile LLVM for `target`. + fn run(self, builder: &Builder<'_>) -> PathBuf { + let target = self.target; + let target_native = if self.target.starts_with("riscv") { + // RISC-V target triples in Rust is not named the same as C compiler target triples. + // This converts Rust RISC-V target triples to C compiler triples. + let idx = target.triple.find('-').unwrap(); + + format!("riscv{}{}", &target.triple[5..7], &target.triple[idx..]) + } else if self.target.starts_with("powerpc") && self.target.ends_with("freebsd") { + // FreeBSD 13 had incompatible ABI changes on all PowerPC platforms. + // Set the version suffix to 13.0 so the correct target details are used. + format!("{}{}", self.target, "13.0") + } else { + target.to_string() + }; + + let Meta { stamp, build_llvm_config, out_dir, root } = + match prebuilt_llvm_config(builder, target) { + Ok(p) => return p, + Err(m) => m, + }; + + builder.update_submodule(&Path::new("src").join("llvm-project")); + if builder.llvm_link_shared() && target.contains("windows") { + panic!("shared linking to LLVM is not currently supported on {}", target.triple); + } + + builder.info(&format!("Building LLVM for {}", target)); + t!(stamp.remove()); + let _time = util::timeit(&builder); + t!(fs::create_dir_all(&out_dir)); + + // https://llvm.org/docs/CMake.html + let mut cfg = cmake::Config::new(builder.src.join(root)); + let mut ldflags = LdFlags::default(); + + let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) { + (false, _) => "Debug", + (true, false) => "Release", + (true, true) => "RelWithDebInfo", + }; + + // NOTE: remember to also update `config.toml.example` when changing the + // defaults! + let llvm_targets = match &builder.config.llvm_targets { + Some(s) => s, + None => { + "AArch64;ARM;BPF;Hexagon;MSP430;Mips;NVPTX;PowerPC;RISCV;\ + Sparc;SystemZ;WebAssembly;X86" + } + }; + + let llvm_exp_targets = match builder.config.llvm_experimental_targets { + Some(ref s) => s, + None => "AVR;M68k", + }; + + let assertions = if builder.config.llvm_assertions { "ON" } else { "OFF" }; + let plugins = if builder.config.llvm_plugins { "ON" } else { "OFF" }; + let enable_tests = if builder.config.llvm_tests { "ON" } else { "OFF" }; + + cfg.out_dir(&out_dir) + .profile(profile) + .define("LLVM_ENABLE_ASSERTIONS", assertions) + .define("LLVM_ENABLE_PLUGINS", plugins) + .define("LLVM_TARGETS_TO_BUILD", llvm_targets) + .define("LLVM_EXPERIMENTAL_TARGETS_TO_BUILD", llvm_exp_targets) + .define("LLVM_INCLUDE_EXAMPLES", "OFF") + .define("LLVM_INCLUDE_DOCS", "OFF") + .define("LLVM_INCLUDE_BENCHMARKS", "OFF") + .define("LLVM_INCLUDE_TESTS", enable_tests) + .define("LLVM_ENABLE_TERMINFO", "OFF") + .define("LLVM_ENABLE_LIBEDIT", "OFF") + .define("LLVM_ENABLE_BINDINGS", "OFF") + .define("LLVM_ENABLE_Z3_SOLVER", "OFF") + .define("LLVM_PARALLEL_COMPILE_JOBS", builder.jobs().to_string()) + .define("LLVM_TARGET_ARCH", target_native.split('-').next().unwrap()) + .define("LLVM_DEFAULT_TARGET_TRIPLE", target_native); + + // Parts of our test suite rely on the `FileCheck` tool, which is built by default in + // `build/$TARGET/llvm/build/bin` is but *not* then installed to `build/$TARGET/llvm/bin`. + // This flag makes sure `FileCheck` is copied in the final binaries directory. + cfg.define("LLVM_INSTALL_UTILS", "ON"); + + if builder.config.llvm_profile_generate { + cfg.define("LLVM_BUILD_INSTRUMENTED", "IR"); + if let Ok(llvm_profile_dir) = std::env::var("LLVM_PROFILE_DIR") { + cfg.define("LLVM_PROFILE_DATA_DIR", llvm_profile_dir); + } + cfg.define("LLVM_BUILD_RUNTIME", "No"); + } + if let Some(path) = builder.config.llvm_profile_use.as_ref() { + cfg.define("LLVM_PROFDATA_FILE", &path); + } + + if target != "aarch64-apple-darwin" && !target.contains("windows") { + cfg.define("LLVM_ENABLE_ZLIB", "ON"); + } else { + cfg.define("LLVM_ENABLE_ZLIB", "OFF"); + } + + // Are we compiling for iOS/tvOS/watchOS? + if target.contains("apple-ios") + || target.contains("apple-tvos") + || target.contains("apple-watchos") + { + // These two defines prevent CMake from automatically trying to add a MacOSX sysroot, which leads to a compiler error. + cfg.define("CMAKE_OSX_SYSROOT", "/"); + cfg.define("CMAKE_OSX_DEPLOYMENT_TARGET", ""); + // Prevent cmake from adding -bundle to CFLAGS automatically, which leads to a compiler error because "-bitcode_bundle" also gets added. + cfg.define("LLVM_ENABLE_PLUGINS", "OFF"); + // Zlib fails to link properly, leading to a compiler error. + cfg.define("LLVM_ENABLE_ZLIB", "OFF"); + } + + // This setting makes the LLVM tools link to the dynamic LLVM library, + // which saves both memory during parallel links and overall disk space + // for the tools. We don't do this on every platform as it doesn't work + // equally well everywhere. + // + // If we're not linking rustc to a dynamic LLVM, though, then don't link + // tools to it. + let llvm_link_shared = + builder.llvm_link_tools_dynamically(target) && builder.llvm_link_shared(); + if llvm_link_shared { + cfg.define("LLVM_LINK_LLVM_DYLIB", "ON"); + } + + if target.starts_with("riscv") && !target.contains("freebsd") { + // RISC-V GCC erroneously requires linking against + // `libatomic` when using 1-byte and 2-byte C++ + // atomics but the LLVM build system check cannot + // detect this. Therefore it is set manually here. + // FreeBSD uses Clang as its system compiler and + // provides no libatomic in its base system so does + // not want this. + ldflags.exe.push(" -latomic"); + ldflags.shared.push(" -latomic"); + } + + if target.contains("msvc") { + cfg.define("LLVM_USE_CRT_DEBUG", "MT"); + cfg.define("LLVM_USE_CRT_RELEASE", "MT"); + cfg.define("LLVM_USE_CRT_RELWITHDEBINFO", "MT"); + cfg.static_crt(true); + } + + if target.starts_with("i686") { + cfg.define("LLVM_BUILD_32_BITS", "ON"); + } + + let mut enabled_llvm_projects = Vec::new(); + + if util::forcing_clang_based_tests() { + enabled_llvm_projects.push("clang"); + enabled_llvm_projects.push("compiler-rt"); + } + + if builder.config.llvm_polly { + enabled_llvm_projects.push("polly"); + } + + if builder.config.llvm_clang { + enabled_llvm_projects.push("clang"); + } + + // We want libxml to be disabled. + // See https://github.com/rust-lang/rust/pull/50104 + cfg.define("LLVM_ENABLE_LIBXML2", "OFF"); + + if !enabled_llvm_projects.is_empty() { + enabled_llvm_projects.sort(); + enabled_llvm_projects.dedup(); + cfg.define("LLVM_ENABLE_PROJECTS", enabled_llvm_projects.join(";")); + } + + if let Some(num_linkers) = builder.config.llvm_link_jobs { + if num_linkers > 0 { + cfg.define("LLVM_PARALLEL_LINK_JOBS", num_linkers.to_string()); + } + } + + // Workaround for ppc32 lld limitation + if target == "powerpc-unknown-freebsd" { + ldflags.exe.push(" -fuse-ld=bfd"); + } + + // https://llvm.org/docs/HowToCrossCompileLLVM.html + if target != builder.config.build { + builder.ensure(Llvm { target: builder.config.build }); + // FIXME: if the llvm root for the build triple is overridden then we + // should use llvm-tblgen from there, also should verify that it + // actually exists most of the time in normal installs of LLVM. + let host_bin = builder.llvm_out(builder.config.build).join("bin"); + cfg.define("LLVM_TABLEGEN", host_bin.join("llvm-tblgen").with_extension(EXE_EXTENSION)); + // LLVM_NM is required for cross compiling using MSVC + cfg.define("LLVM_NM", host_bin.join("llvm-nm").with_extension(EXE_EXTENSION)); + cfg.define( + "LLVM_CONFIG_PATH", + host_bin.join("llvm-config").with_extension(EXE_EXTENSION), + ); + if builder.config.llvm_clang { + let build_bin = builder.llvm_out(builder.config.build).join("build").join("bin"); + let clang_tblgen = build_bin.join("clang-tblgen").with_extension(EXE_EXTENSION); + if !builder.config.dry_run && !clang_tblgen.exists() { + panic!("unable to find {}", clang_tblgen.display()); + } + cfg.define("CLANG_TABLEGEN", clang_tblgen); + } + } + + let llvm_version_suffix = if let Some(ref suffix) = builder.config.llvm_version_suffix { + // Allow version-suffix="" to not define a version suffix at all. + if !suffix.is_empty() { Some(suffix.to_string()) } else { None } + } else if builder.config.channel == "dev" { + // Changes to a version suffix require a complete rebuild of the LLVM. + // To avoid rebuilds during a time of version bump, don't include rustc + // release number on the dev channel. + Some("-rust-dev".to_string()) + } else { + Some(format!("-rust-{}-{}", builder.version, builder.config.channel)) + }; + if let Some(ref suffix) = llvm_version_suffix { + cfg.define("LLVM_VERSION_SUFFIX", suffix); + } + + configure_cmake(builder, target, &mut cfg, true, ldflags); + configure_llvm(builder, target, &mut cfg); + + for (key, val) in &builder.config.llvm_build_config { + cfg.define(key, val); + } + + // FIXME: we don't actually need to build all LLVM tools and all LLVM + // libraries here, e.g., we just want a few components and a few + // tools. Figure out how to filter them down and only build the right + // tools and libs on all platforms. + + if builder.config.dry_run { + return build_llvm_config; + } + + cfg.build(); + + // When building LLVM with LLVM_LINK_LLVM_DYLIB for macOS, an unversioned + // libLLVM.dylib will be built. However, llvm-config will still look + // for a versioned path like libLLVM-14.dylib. Manually create a symbolic + // link to make llvm-config happy. + if llvm_link_shared && target.contains("apple-darwin") { + let mut cmd = Command::new(&build_llvm_config); + let version = output(cmd.arg("--version")); + let major = version.split('.').next().unwrap(); + let lib_name = match llvm_version_suffix { + Some(s) => format!("libLLVM-{}{}.dylib", major, s), + None => format!("libLLVM-{}.dylib", major), + }; + + let lib_llvm = out_dir.join("build").join("lib").join(lib_name); + if !lib_llvm.exists() { + t!(builder.symlink_file("libLLVM.dylib", &lib_llvm)); + } + } + + t!(stamp.write()); + + build_llvm_config + } +} + +fn check_llvm_version(builder: &Builder<'_>, llvm_config: &Path) { + if !builder.config.llvm_version_check { + return; + } + + if builder.config.dry_run { + return; + } + + let mut cmd = Command::new(llvm_config); + let version = output(cmd.arg("--version")); + let mut parts = version.split('.').take(2).filter_map(|s| s.parse::<u32>().ok()); + if let (Some(major), Some(_minor)) = (parts.next(), parts.next()) { + if major >= 12 { + return; + } + } + panic!("\n\nbad LLVM version: {}, need >=12.0\n\n", version) +} + +fn configure_cmake( + builder: &Builder<'_>, + target: TargetSelection, + cfg: &mut cmake::Config, + use_compiler_launcher: bool, + mut ldflags: LdFlags, +) { + // Do not print installation messages for up-to-date files. + // LLVM and LLD builds can produce a lot of those and hit CI limits on log size. + cfg.define("CMAKE_INSTALL_MESSAGE", "LAZY"); + + // Do not allow the user's value of DESTDIR to influence where + // LLVM will install itself. LLVM must always be installed in our + // own build directories. + cfg.env("DESTDIR", ""); + + if builder.ninja() { + cfg.generator("Ninja"); + } + cfg.target(&target.triple).host(&builder.config.build.triple); + + if target != builder.config.build { + cfg.define("CMAKE_CROSSCOMPILING", "True"); + + if target.contains("netbsd") { + cfg.define("CMAKE_SYSTEM_NAME", "NetBSD"); + } else if target.contains("freebsd") { + cfg.define("CMAKE_SYSTEM_NAME", "FreeBSD"); + } else if target.contains("windows") { + cfg.define("CMAKE_SYSTEM_NAME", "Windows"); + } else if target.contains("haiku") { + cfg.define("CMAKE_SYSTEM_NAME", "Haiku"); + } else if target.contains("solaris") || target.contains("illumos") { + cfg.define("CMAKE_SYSTEM_NAME", "SunOS"); + } + // When cross-compiling we should also set CMAKE_SYSTEM_VERSION, but in + // that case like CMake we cannot easily determine system version either. + // + // Since, the LLVM itself makes rather limited use of version checks in + // CMakeFiles (and then only in tests), and so far no issues have been + // reported, the system version is currently left unset. + + if target.contains("darwin") { + // Make sure that CMake does not build universal binaries on macOS. + // Explicitly specifiy the one single target architecture. + if target.starts_with("aarch64") { + // macOS uses a different name for building arm64 + cfg.define("CMAKE_OSX_ARCHITECTURES", "arm64"); + } else { + cfg.define("CMAKE_OSX_ARCHITECTURES", target.triple.split('-').next().unwrap()); + } + } + } + + let sanitize_cc = |cc: &Path| { + if target.contains("msvc") { + OsString::from(cc.to_str().unwrap().replace("\\", "/")) + } else { + cc.as_os_str().to_owned() + } + }; + + // MSVC with CMake uses msbuild by default which doesn't respect these + // vars that we'd otherwise configure. In that case we just skip this + // entirely. + if target.contains("msvc") && !builder.ninja() { + return; + } + + let (cc, cxx) = match builder.config.llvm_clang_cl { + Some(ref cl) => (cl.as_ref(), cl.as_ref()), + None => (builder.cc(target), builder.cxx(target).unwrap()), + }; + + // Handle msvc + ninja + ccache specially (this is what the bots use) + if target.contains("msvc") && builder.ninja() && builder.config.ccache.is_some() { + let mut wrap_cc = env::current_exe().expect("failed to get cwd"); + wrap_cc.set_file_name("sccache-plus-cl.exe"); + + cfg.define("CMAKE_C_COMPILER", sanitize_cc(&wrap_cc)) + .define("CMAKE_CXX_COMPILER", sanitize_cc(&wrap_cc)); + cfg.env("SCCACHE_PATH", builder.config.ccache.as_ref().unwrap()) + .env("SCCACHE_TARGET", target.triple) + .env("SCCACHE_CC", &cc) + .env("SCCACHE_CXX", &cxx); + + // Building LLVM on MSVC can be a little ludicrous at times. We're so far + // off the beaten path here that I'm not really sure this is even half + // supported any more. Here we're trying to: + // + // * Build LLVM on MSVC + // * Build LLVM with `clang-cl` instead of `cl.exe` + // * Build a project with `sccache` + // * Build for 32-bit as well + // * Build with Ninja + // + // For `cl.exe` there are different binaries to compile 32/64 bit which + // we use but for `clang-cl` there's only one which internally + // multiplexes via flags. As a result it appears that CMake's detection + // of a compiler's architecture and such on MSVC **doesn't** pass any + // custom flags we pass in CMAKE_CXX_FLAGS below. This means that if we + // use `clang-cl.exe` it's always diagnosed as a 64-bit compiler which + // definitely causes problems since all the env vars are pointing to + // 32-bit libraries. + // + // To hack around this... again... we pass an argument that's + // unconditionally passed in the sccache shim. This'll get CMake to + // correctly diagnose it's doing a 32-bit compilation and LLVM will + // internally configure itself appropriately. + if builder.config.llvm_clang_cl.is_some() && target.contains("i686") { + cfg.env("SCCACHE_EXTRA_ARGS", "-m32"); + } + } else { + // If ccache is configured we inform the build a little differently how + // to invoke ccache while also invoking our compilers. + if use_compiler_launcher { + if let Some(ref ccache) = builder.config.ccache { + cfg.define("CMAKE_C_COMPILER_LAUNCHER", ccache) + .define("CMAKE_CXX_COMPILER_LAUNCHER", ccache); + } + } + cfg.define("CMAKE_C_COMPILER", sanitize_cc(cc)) + .define("CMAKE_CXX_COMPILER", sanitize_cc(cxx)) + .define("CMAKE_ASM_COMPILER", sanitize_cc(cc)); + } + + cfg.build_arg("-j").build_arg(builder.jobs().to_string()); + let mut cflags: OsString = builder.cflags(target, GitRepo::Llvm, CLang::C).join(" ").into(); + if let Some(ref s) = builder.config.llvm_cflags { + cflags.push(" "); + cflags.push(s); + } + // Some compiler features used by LLVM (such as thread locals) will not work on a min version below iOS 10. + if target.contains("apple-ios") { + if target.contains("86-") { + cflags.push(" -miphonesimulator-version-min=10.0"); + } else { + cflags.push(" -miphoneos-version-min=10.0"); + } + } + if builder.config.llvm_clang_cl.is_some() { + cflags.push(&format!(" --target={}", target)); + } + cfg.define("CMAKE_C_FLAGS", cflags); + let mut cxxflags: OsString = builder.cflags(target, GitRepo::Llvm, CLang::Cxx).join(" ").into(); + if let Some(ref s) = builder.config.llvm_cxxflags { + cxxflags.push(" "); + cxxflags.push(s); + } + if builder.config.llvm_clang_cl.is_some() { + cxxflags.push(&format!(" --target={}", target)); + } + cfg.define("CMAKE_CXX_FLAGS", cxxflags); + if let Some(ar) = builder.ar(target) { + if ar.is_absolute() { + // LLVM build breaks if `CMAKE_AR` is a relative path, for some reason it + // tries to resolve this path in the LLVM build directory. + cfg.define("CMAKE_AR", sanitize_cc(ar)); + } + } + + if let Some(ranlib) = builder.ranlib(target) { + if ranlib.is_absolute() { + // LLVM build breaks if `CMAKE_RANLIB` is a relative path, for some reason it + // tries to resolve this path in the LLVM build directory. + cfg.define("CMAKE_RANLIB", sanitize_cc(ranlib)); + } + } + + if let Some(ref flags) = builder.config.llvm_ldflags { + ldflags.push_all(flags); + } + + if let Some(flags) = get_var("LDFLAGS", &builder.config.build.triple, &target.triple) { + ldflags.push_all(&flags); + } + + // For distribution we want the LLVM tools to be *statically* linked to libstdc++. + // We also do this if the user explicitly requested static libstdc++. + if builder.config.llvm_static_stdcpp { + if !target.contains("msvc") && !target.contains("netbsd") && !target.contains("solaris") { + if target.contains("apple") || target.contains("windows") { + ldflags.push_all("-static-libstdc++"); + } else { + ldflags.push_all("-Wl,-Bsymbolic -static-libstdc++"); + } + } + } + + cfg.define("CMAKE_SHARED_LINKER_FLAGS", &ldflags.shared); + cfg.define("CMAKE_MODULE_LINKER_FLAGS", &ldflags.module); + cfg.define("CMAKE_EXE_LINKER_FLAGS", &ldflags.exe); + + if env::var_os("SCCACHE_ERROR_LOG").is_some() { + cfg.env("RUSTC_LOG", "sccache=warn"); + } +} + +fn configure_llvm(builder: &Builder<'_>, target: TargetSelection, cfg: &mut cmake::Config) { + // ThinLTO is only available when building with LLVM, enabling LLD is required. + // Apple's linker ld64 supports ThinLTO out of the box though, so don't use LLD on Darwin. + if builder.config.llvm_thin_lto { + cfg.define("LLVM_ENABLE_LTO", "Thin"); + if !target.contains("apple") { + cfg.define("LLVM_ENABLE_LLD", "ON"); + } + } + + if let Some(ref linker) = builder.config.llvm_use_linker { + cfg.define("LLVM_USE_LINKER", linker); + } + + if builder.config.llvm_allow_old_toolchain { + cfg.define("LLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN", "YES"); + } +} + +// Adapted from https://github.com/alexcrichton/cc-rs/blob/fba7feded71ee4f63cfe885673ead6d7b4f2f454/src/lib.rs#L2347-L2365 +fn get_var(var_base: &str, host: &str, target: &str) -> Option<OsString> { + let kind = if host == target { "HOST" } else { "TARGET" }; + let target_u = target.replace("-", "_"); + env::var_os(&format!("{}_{}", var_base, target)) + .or_else(|| env::var_os(&format!("{}_{}", var_base, target_u))) + .or_else(|| env::var_os(&format!("{}_{}", kind, var_base))) + .or_else(|| env::var_os(var_base)) +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Lld { + pub target: TargetSelection, +} + +impl Step for Lld { + type Output = PathBuf; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/llvm-project/lld") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Lld { target: run.target }); + } + + /// Compile LLD for `target`. + fn run(self, builder: &Builder<'_>) -> PathBuf { + if builder.config.dry_run { + return PathBuf::from("lld-out-dir-test-gen"); + } + let target = self.target; + + let llvm_config = builder.ensure(Llvm { target: self.target }); + + let out_dir = builder.lld_out(target); + let done_stamp = out_dir.join("lld-finished-building"); + if done_stamp.exists() { + return out_dir; + } + + builder.info(&format!("Building LLD for {}", target)); + let _time = util::timeit(&builder); + t!(fs::create_dir_all(&out_dir)); + + let mut cfg = cmake::Config::new(builder.src.join("src/llvm-project/lld")); + let mut ldflags = LdFlags::default(); + + // When building LLD as part of a build with instrumentation on windows, for example + // when doing PGO on CI, cmake or clang-cl don't automatically link clang's + // profiler runtime in. In that case, we need to manually ask cmake to do it, to avoid + // linking errors, much like LLVM's cmake setup does in that situation. + if builder.config.llvm_profile_generate && target.contains("msvc") { + if let Some(clang_cl_path) = builder.config.llvm_clang_cl.as_ref() { + // Find clang's runtime library directory and push that as a search path to the + // cmake linker flags. + let clang_rt_dir = get_clang_cl_resource_dir(clang_cl_path); + ldflags.push_all(&format!("/libpath:{}", clang_rt_dir.display())); + } + } + + configure_cmake(builder, target, &mut cfg, true, ldflags); + configure_llvm(builder, target, &mut cfg); + + // This is an awful, awful hack. Discovered when we migrated to using + // clang-cl to compile LLVM/LLD it turns out that LLD, when built out of + // tree, will execute `llvm-config --cmakedir` and then tell CMake about + // that directory for later processing. Unfortunately if this path has + // forward slashes in it (which it basically always does on Windows) + // then CMake will hit a syntax error later on as... something isn't + // escaped it seems? + // + // Instead of attempting to fix this problem in upstream CMake and/or + // LLVM/LLD we just hack around it here. This thin wrapper will take the + // output from llvm-config and replace all instances of `\` with `/` to + // ensure we don't hit the same bugs with escaping. It means that you + // can't build on a system where your paths require `\` on Windows, but + // there's probably a lot of reasons you can't do that other than this. + let llvm_config_shim = env::current_exe().unwrap().with_file_name("llvm-config-wrapper"); + + // Re-use the same flags as llvm to control the level of debug information + // generated for lld. + let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) { + (false, _) => "Debug", + (true, false) => "Release", + (true, true) => "RelWithDebInfo", + }; + + cfg.out_dir(&out_dir) + .profile(profile) + .env("LLVM_CONFIG_REAL", &llvm_config) + .define("LLVM_CONFIG_PATH", llvm_config_shim) + .define("LLVM_INCLUDE_TESTS", "OFF"); + + // While we're using this horrible workaround to shim the execution of + // llvm-config, let's just pile on more. I can't seem to figure out how + // to build LLD as a standalone project and also cross-compile it at the + // same time. It wants a natively executable `llvm-config` to learn + // about LLVM, but then it learns about all the host configuration of + // LLVM and tries to link to host LLVM libraries. + // + // To work around that we tell our shim to replace anything with the + // build target with the actual target instead. This'll break parts of + // LLD though which try to execute host tools, such as llvm-tblgen, so + // we specifically tell it where to find those. This is likely super + // brittle and will break over time. If anyone knows better how to + // cross-compile LLD it would be much appreciated to fix this! + if target != builder.config.build { + cfg.env("LLVM_CONFIG_SHIM_REPLACE", &builder.config.build.triple) + .env("LLVM_CONFIG_SHIM_REPLACE_WITH", &target.triple) + .define( + "LLVM_TABLEGEN_EXE", + llvm_config.with_file_name("llvm-tblgen").with_extension(EXE_EXTENSION), + ); + } + + // Explicitly set C++ standard, because upstream doesn't do so + // for standalone builds. + cfg.define("CMAKE_CXX_STANDARD", "14"); + + cfg.build(); + + t!(File::create(&done_stamp)); + out_dir + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct TestHelpers { + pub target: TargetSelection, +} + +impl Step for TestHelpers { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/test/auxiliary/rust_test_helpers.c") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(TestHelpers { target: run.target }) + } + + /// Compiles the `rust_test_helpers.c` library which we used in various + /// `run-pass` tests for ABI testing. + fn run(self, builder: &Builder<'_>) { + if builder.config.dry_run { + return; + } + // The x86_64-fortanix-unknown-sgx target doesn't have a working C + // toolchain. However, some x86_64 ELF objects can be linked + // without issues. Use this hack to compile the test helpers. + let target = if self.target == "x86_64-fortanix-unknown-sgx" { + TargetSelection::from_user("x86_64-unknown-linux-gnu") + } else { + self.target + }; + let dst = builder.test_helpers_out(target); + let src = builder.src.join("src/test/auxiliary/rust_test_helpers.c"); + if up_to_date(&src, &dst.join("librust_test_helpers.a")) { + return; + } + + builder.info("Building test helpers"); + t!(fs::create_dir_all(&dst)); + let mut cfg = cc::Build::new(); + // FIXME: Workaround for https://github.com/emscripten-core/emscripten/issues/9013 + if target.contains("emscripten") { + cfg.pic(false); + } + + // We may have found various cross-compilers a little differently due to our + // extra configuration, so inform cc of these compilers. Note, though, that + // on MSVC we still need cc's detection of env vars (ugh). + if !target.contains("msvc") { + if let Some(ar) = builder.ar(target) { + cfg.archiver(ar); + } + cfg.compiler(builder.cc(target)); + } + cfg.cargo_metadata(false) + .out_dir(&dst) + .target(&target.triple) + .host(&builder.config.build.triple) + .opt_level(0) + .warnings(false) + .debug(false) + .file(builder.src.join("src/test/auxiliary/rust_test_helpers.c")) + .compile("rust_test_helpers"); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Sanitizers { + pub target: TargetSelection, +} + +impl Step for Sanitizers { + type Output = Vec<SanitizerRuntime>; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("sanitizers") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Sanitizers { target: run.target }); + } + + /// Builds sanitizer runtime libraries. + fn run(self, builder: &Builder<'_>) -> Self::Output { + let compiler_rt_dir = builder.src.join("src/llvm-project/compiler-rt"); + if !compiler_rt_dir.exists() { + return Vec::new(); + } + + let out_dir = builder.native_dir(self.target).join("sanitizers"); + let runtimes = supported_sanitizers(&out_dir, self.target, &builder.config.channel); + if runtimes.is_empty() { + return runtimes; + } + + let llvm_config = builder.ensure(Llvm { target: builder.config.build }); + if builder.config.dry_run { + return runtimes; + } + + let stamp = out_dir.join("sanitizers-finished-building"); + let stamp = HashStamp::new(stamp, builder.in_tree_llvm_info.sha()); + + if stamp.is_done() { + if stamp.hash.is_none() { + builder.info(&format!( + "Rebuild sanitizers by removing the file `{}`", + stamp.path.display() + )); + } + return runtimes; + } + + builder.info(&format!("Building sanitizers for {}", self.target)); + t!(stamp.remove()); + let _time = util::timeit(&builder); + + let mut cfg = cmake::Config::new(&compiler_rt_dir); + cfg.profile("Release"); + cfg.define("CMAKE_C_COMPILER_TARGET", self.target.triple); + cfg.define("COMPILER_RT_BUILD_BUILTINS", "OFF"); + cfg.define("COMPILER_RT_BUILD_CRT", "OFF"); + cfg.define("COMPILER_RT_BUILD_LIBFUZZER", "OFF"); + cfg.define("COMPILER_RT_BUILD_PROFILE", "OFF"); + cfg.define("COMPILER_RT_BUILD_SANITIZERS", "ON"); + cfg.define("COMPILER_RT_BUILD_XRAY", "OFF"); + cfg.define("COMPILER_RT_DEFAULT_TARGET_ONLY", "ON"); + cfg.define("COMPILER_RT_USE_LIBCXX", "OFF"); + cfg.define("LLVM_CONFIG_PATH", &llvm_config); + + // On Darwin targets the sanitizer runtimes are build as universal binaries. + // Unfortunately sccache currently lacks support to build them successfully. + // Disable compiler launcher on Darwin targets to avoid potential issues. + let use_compiler_launcher = !self.target.contains("apple-darwin"); + configure_cmake(builder, self.target, &mut cfg, use_compiler_launcher, LdFlags::default()); + + t!(fs::create_dir_all(&out_dir)); + cfg.out_dir(out_dir); + + for runtime in &runtimes { + cfg.build_target(&runtime.cmake_target); + cfg.build(); + } + t!(stamp.write()); + + runtimes + } +} + +#[derive(Clone, Debug)] +pub struct SanitizerRuntime { + /// CMake target used to build the runtime. + pub cmake_target: String, + /// Path to the built runtime library. + pub path: PathBuf, + /// Library filename that will be used rustc. + pub name: String, +} + +/// Returns sanitizers available on a given target. +fn supported_sanitizers( + out_dir: &Path, + target: TargetSelection, + channel: &str, +) -> Vec<SanitizerRuntime> { + let darwin_libs = |os: &str, components: &[&str]| -> Vec<SanitizerRuntime> { + components + .iter() + .map(move |c| SanitizerRuntime { + cmake_target: format!("clang_rt.{}_{}_dynamic", c, os), + path: out_dir + .join(&format!("build/lib/darwin/libclang_rt.{}_{}_dynamic.dylib", c, os)), + name: format!("librustc-{}_rt.{}.dylib", channel, c), + }) + .collect() + }; + + let common_libs = |os: &str, arch: &str, components: &[&str]| -> Vec<SanitizerRuntime> { + components + .iter() + .map(move |c| SanitizerRuntime { + cmake_target: format!("clang_rt.{}-{}", c, arch), + path: out_dir.join(&format!("build/lib/{}/libclang_rt.{}-{}.a", os, c, arch)), + name: format!("librustc-{}_rt.{}.a", channel, c), + }) + .collect() + }; + + match &*target.triple { + "aarch64-apple-darwin" => darwin_libs("osx", &["asan", "lsan", "tsan"]), + "aarch64-fuchsia" => common_libs("fuchsia", "aarch64", &["asan"]), + "aarch64-unknown-linux-gnu" => { + common_libs("linux", "aarch64", &["asan", "lsan", "msan", "tsan", "hwasan"]) + } + "x86_64-apple-darwin" => darwin_libs("osx", &["asan", "lsan", "tsan"]), + "x86_64-fuchsia" => common_libs("fuchsia", "x86_64", &["asan"]), + "x86_64-unknown-freebsd" => common_libs("freebsd", "x86_64", &["asan", "msan", "tsan"]), + "x86_64-unknown-netbsd" => { + common_libs("netbsd", "x86_64", &["asan", "lsan", "msan", "tsan"]) + } + "x86_64-unknown-illumos" => common_libs("illumos", "x86_64", &["asan"]), + "x86_64-pc-solaris" => common_libs("solaris", "x86_64", &["asan"]), + "x86_64-unknown-linux-gnu" => { + common_libs("linux", "x86_64", &["asan", "lsan", "msan", "tsan"]) + } + "x86_64-unknown-linux-musl" => { + common_libs("linux", "x86_64", &["asan", "lsan", "msan", "tsan"]) + } + _ => Vec::new(), + } +} + +struct HashStamp { + path: PathBuf, + hash: Option<Vec<u8>>, +} + +impl HashStamp { + fn new(path: PathBuf, hash: Option<&str>) -> Self { + HashStamp { path, hash: hash.map(|s| s.as_bytes().to_owned()) } + } + + fn is_done(&self) -> bool { + match fs::read(&self.path) { + Ok(h) => self.hash.as_deref().unwrap_or(b"") == h.as_slice(), + Err(e) if e.kind() == io::ErrorKind::NotFound => false, + Err(e) => { + panic!("failed to read stamp file `{}`: {}", self.path.display(), e); + } + } + } + + fn remove(&self) -> io::Result<()> { + match fs::remove_file(&self.path) { + Ok(()) => Ok(()), + Err(e) => { + if e.kind() == io::ErrorKind::NotFound { + Ok(()) + } else { + Err(e) + } + } + } + } + + fn write(&self) -> io::Result<()> { + fs::write(&self.path, self.hash.as_deref().unwrap_or(b"")) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct CrtBeginEnd { + pub target: TargetSelection, +} + +impl Step for CrtBeginEnd { + type Output = PathBuf; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/llvm-project/compiler-rt/lib/crt") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(CrtBeginEnd { target: run.target }); + } + + /// Build crtbegin.o/crtend.o for musl target. + fn run(self, builder: &Builder<'_>) -> Self::Output { + let out_dir = builder.native_dir(self.target).join("crt"); + + if builder.config.dry_run { + return out_dir; + } + + let crtbegin_src = builder.src.join("src/llvm-project/compiler-rt/lib/crt/crtbegin.c"); + let crtend_src = builder.src.join("src/llvm-project/compiler-rt/lib/crt/crtend.c"); + if up_to_date(&crtbegin_src, &out_dir.join("crtbegin.o")) + && up_to_date(&crtend_src, &out_dir.join("crtendS.o")) + { + return out_dir; + } + + builder.info("Building crtbegin.o and crtend.o"); + t!(fs::create_dir_all(&out_dir)); + + let mut cfg = cc::Build::new(); + + if let Some(ar) = builder.ar(self.target) { + cfg.archiver(ar); + } + cfg.compiler(builder.cc(self.target)); + cfg.cargo_metadata(false) + .out_dir(&out_dir) + .target(&self.target.triple) + .host(&builder.config.build.triple) + .warnings(false) + .debug(false) + .opt_level(3) + .file(crtbegin_src) + .file(crtend_src); + + // Those flags are defined in src/llvm-project/compiler-rt/lib/crt/CMakeLists.txt + // Currently only consumer of those objects is musl, which use .init_array/.fini_array + // instead of .ctors/.dtors + cfg.flag("-std=c11") + .define("CRT_HAS_INITFINI_ARRAY", None) + .define("EH_USE_FRAME_REGISTRY", None); + + cfg.compile("crt"); + + t!(fs::copy(out_dir.join("crtbegin.o"), out_dir.join("crtbeginS.o"))); + t!(fs::copy(out_dir.join("crtend.o"), out_dir.join("crtendS.o"))); + out_dir + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Libunwind { + pub target: TargetSelection, +} + +impl Step for Libunwind { + type Output = PathBuf; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/llvm-project/libunwind") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Libunwind { target: run.target }); + } + + /// Build linunwind.a + fn run(self, builder: &Builder<'_>) -> Self::Output { + if builder.config.dry_run { + return PathBuf::new(); + } + + let out_dir = builder.native_dir(self.target).join("libunwind"); + let root = builder.src.join("src/llvm-project/libunwind"); + + if up_to_date(&root, &out_dir.join("libunwind.a")) { + return out_dir; + } + + builder.info(&format!("Building libunwind.a for {}", self.target.triple)); + t!(fs::create_dir_all(&out_dir)); + + let mut cc_cfg = cc::Build::new(); + let mut cpp_cfg = cc::Build::new(); + + cpp_cfg.cpp(true); + cpp_cfg.cpp_set_stdlib(None); + cpp_cfg.flag("-nostdinc++"); + cpp_cfg.flag("-fno-exceptions"); + cpp_cfg.flag("-fno-rtti"); + cpp_cfg.flag_if_supported("-fvisibility-global-new-delete-hidden"); + + for cfg in [&mut cc_cfg, &mut cpp_cfg].iter_mut() { + if let Some(ar) = builder.ar(self.target) { + cfg.archiver(ar); + } + cfg.target(&self.target.triple); + cfg.host(&builder.config.build.triple); + cfg.warnings(false); + cfg.debug(false); + // get_compiler() need set opt_level first. + cfg.opt_level(3); + cfg.flag("-fstrict-aliasing"); + cfg.flag("-funwind-tables"); + cfg.flag("-fvisibility=hidden"); + cfg.define("_LIBUNWIND_DISABLE_VISIBILITY_ANNOTATIONS", None); + cfg.include(root.join("include")); + cfg.cargo_metadata(false); + cfg.out_dir(&out_dir); + + if self.target.contains("x86_64-fortanix-unknown-sgx") { + cfg.static_flag(true); + cfg.flag("-fno-stack-protector"); + cfg.flag("-ffreestanding"); + cfg.flag("-fexceptions"); + + // easiest way to undefine since no API available in cc::Build to undefine + cfg.flag("-U_FORTIFY_SOURCE"); + cfg.define("_FORTIFY_SOURCE", "0"); + cfg.define("RUST_SGX", "1"); + cfg.define("__NO_STRING_INLINES", None); + cfg.define("__NO_MATH_INLINES", None); + cfg.define("_LIBUNWIND_IS_BAREMETAL", None); + cfg.define("__LIBUNWIND_IS_NATIVE_ONLY", None); + cfg.define("NDEBUG", None); + } + if self.target.contains("windows") { + cfg.define("_LIBUNWIND_HIDE_SYMBOLS", "1"); + cfg.define("_LIBUNWIND_IS_NATIVE_ONLY", "1"); + } + } + + cc_cfg.compiler(builder.cc(self.target)); + if let Ok(cxx) = builder.cxx(self.target) { + cpp_cfg.compiler(cxx); + } else { + cc_cfg.compiler(builder.cc(self.target)); + } + + // Don't set this for clang + // By default, Clang builds C code in GNU C17 mode. + // By default, Clang builds C++ code according to the C++98 standard, + // with many C++11 features accepted as extensions. + if cc_cfg.get_compiler().is_like_gnu() { + cc_cfg.flag("-std=c99"); + } + if cpp_cfg.get_compiler().is_like_gnu() { + cpp_cfg.flag("-std=c++11"); + } + + if self.target.contains("x86_64-fortanix-unknown-sgx") || self.target.contains("musl") { + // use the same GCC C compiler command to compile C++ code so we do not need to setup the + // C++ compiler env variables on the builders. + // Don't set this for clang++, as clang++ is able to compile this without libc++. + if cpp_cfg.get_compiler().is_like_gnu() { + cpp_cfg.cpp(false); + cpp_cfg.compiler(builder.cc(self.target)); + } + } + + let mut c_sources = vec![ + "Unwind-sjlj.c", + "UnwindLevel1-gcc-ext.c", + "UnwindLevel1.c", + "UnwindRegistersRestore.S", + "UnwindRegistersSave.S", + ]; + + let cpp_sources = vec!["Unwind-EHABI.cpp", "Unwind-seh.cpp", "libunwind.cpp"]; + let cpp_len = cpp_sources.len(); + + if self.target.contains("x86_64-fortanix-unknown-sgx") { + c_sources.push("UnwindRustSgx.c"); + } + + for src in c_sources { + cc_cfg.file(root.join("src").join(src).canonicalize().unwrap()); + } + + for src in &cpp_sources { + cpp_cfg.file(root.join("src").join(src).canonicalize().unwrap()); + } + + cpp_cfg.compile("unwind-cpp"); + + // FIXME: https://github.com/alexcrichton/cc-rs/issues/545#issuecomment-679242845 + let mut count = 0; + for entry in fs::read_dir(&out_dir).unwrap() { + let file = entry.unwrap().path().canonicalize().unwrap(); + if file.is_file() && file.extension() == Some(OsStr::new("o")) { + // file name starts with "Unwind-EHABI", "Unwind-seh" or "libunwind" + let file_name = file.file_name().unwrap().to_str().expect("UTF-8 file name"); + if cpp_sources.iter().any(|f| file_name.starts_with(&f[..f.len() - 4])) { + cc_cfg.object(&file); + count += 1; + } + } + } + assert_eq!(cpp_len, count, "Can't get object files from {:?}", &out_dir); + + cc_cfg.compile("unwind"); + out_dir + } +} diff --git a/src/bootstrap/run.rs b/src/bootstrap/run.rs new file mode 100644 index 000000000..25abe7a72 --- /dev/null +++ b/src/bootstrap/run.rs @@ -0,0 +1,105 @@ +use crate::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::dist::distdir; +use crate::tool::Tool; +use crate::util::output; +use std::process::Command; + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct ExpandYamlAnchors; + +impl Step for ExpandYamlAnchors { + type Output = (); + + /// Runs the `expand-yaml_anchors` tool. + /// + /// This tool in `src/tools` reads the CI configuration files written in YAML and expands the + /// anchors in them, since GitHub Actions doesn't support them. + fn run(self, builder: &Builder<'_>) { + builder.info("Expanding YAML anchors in the GitHub Actions configuration"); + try_run( + builder, + &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("generate").arg(&builder.src), + ); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/expand-yaml-anchors") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(ExpandYamlAnchors); + } +} + +fn try_run(builder: &Builder<'_>, cmd: &mut Command) -> bool { + if !builder.fail_fast { + if !builder.try_run(cmd) { + let mut failures = builder.delayed_failures.borrow_mut(); + failures.push(format!("{:?}", cmd)); + return false; + } + } else { + builder.run(cmd); + } + true +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct BuildManifest; + +impl Step for BuildManifest { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/build-manifest") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(BuildManifest); + } + + fn run(self, builder: &Builder<'_>) { + // This gets called by `promote-release` + // (https://github.com/rust-lang/promote-release). + let mut cmd = builder.tool_cmd(Tool::BuildManifest); + let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| { + panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n") + }); + let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| { + panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n") + }); + + let today = output(Command::new("date").arg("+%Y-%m-%d")); + + cmd.arg(sign); + cmd.arg(distdir(builder)); + cmd.arg(today.trim()); + cmd.arg(addr); + cmd.arg(&builder.config.channel); + + builder.create_dir(&distdir(builder)); + builder.run(&mut cmd); + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct BumpStage0; + +impl Step for BumpStage0 { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/bump-stage0") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(BumpStage0); + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + let mut cmd = builder.tool_cmd(Tool::BumpStage0); + builder.run(&mut cmd); + } +} diff --git a/src/bootstrap/sanity.rs b/src/bootstrap/sanity.rs new file mode 100644 index 000000000..cae41286f --- /dev/null +++ b/src/bootstrap/sanity.rs @@ -0,0 +1,242 @@ +//! Sanity checking performed by rustbuild before actually executing anything. +//! +//! This module contains the implementation of ensuring that the build +//! environment looks reasonable before progressing. This will verify that +//! various programs like git and python exist, along with ensuring that all C +//! compilers for cross-compiling are found. +//! +//! In theory if we get past this phase it's a bug if a build fails, but in +//! practice that's likely not true! + +use std::collections::HashMap; +use std::env; +use std::ffi::{OsStr, OsString}; +use std::fs; +use std::path::PathBuf; +use std::process::Command; + +use crate::cache::INTERNER; +use crate::config::Target; +use crate::util::output; +use crate::Build; + +pub struct Finder { + cache: HashMap<OsString, Option<PathBuf>>, + path: OsString, +} + +impl Finder { + pub fn new() -> Self { + Self { cache: HashMap::new(), path: env::var_os("PATH").unwrap_or_default() } + } + + pub fn maybe_have<S: Into<OsString>>(&mut self, cmd: S) -> Option<PathBuf> { + let cmd: OsString = cmd.into(); + let path = &self.path; + self.cache + .entry(cmd.clone()) + .or_insert_with(|| { + for path in env::split_paths(path) { + let target = path.join(&cmd); + let mut cmd_exe = cmd.clone(); + cmd_exe.push(".exe"); + + if target.is_file() // some/path/git + || path.join(&cmd_exe).exists() // some/path/git.exe + || target.join(&cmd_exe).exists() + // some/path/git/git.exe + { + return Some(target); + } + } + None + }) + .clone() + } + + pub fn must_have<S: AsRef<OsStr>>(&mut self, cmd: S) -> PathBuf { + self.maybe_have(&cmd).unwrap_or_else(|| { + panic!("\n\ncouldn't find required command: {:?}\n\n", cmd.as_ref()); + }) + } +} + +pub fn check(build: &mut Build) { + let path = env::var_os("PATH").unwrap_or_default(); + // On Windows, quotes are invalid characters for filename paths, and if + // one is present as part of the PATH then that can lead to the system + // being unable to identify the files properly. See + // https://github.com/rust-lang/rust/issues/34959 for more details. + if cfg!(windows) && path.to_string_lossy().contains('\"') { + panic!("PATH contains invalid character '\"'"); + } + + let mut cmd_finder = Finder::new(); + // If we've got a git directory we're gonna need git to update + // submodules and learn about various other aspects. + if build.rust_info.is_git() { + cmd_finder.must_have("git"); + } + + // We need cmake, but only if we're actually building LLVM or sanitizers. + let building_llvm = build.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) + && build + .hosts + .iter() + .map(|host| { + build + .config + .target_config + .get(host) + .map(|config| config.llvm_config.is_none()) + .unwrap_or(true) + }) + .any(|build_llvm_ourselves| build_llvm_ourselves); + let need_cmake = building_llvm || build.config.any_sanitizers_enabled(); + if need_cmake { + if cmd_finder.maybe_have("cmake").is_none() { + eprintln!( + " +Couldn't find required command: cmake + +You should install cmake, or set `download-ci-llvm = true` in the +`[llvm]` section section of `config.toml` to download LLVM rather +than building it. +" + ); + crate::detail_exit(1); + } + } + + build.config.python = build + .config + .python + .take() + .map(|p| cmd_finder.must_have(p)) + .or_else(|| env::var_os("BOOTSTRAP_PYTHON").map(PathBuf::from)) // set by bootstrap.py + .or_else(|| cmd_finder.maybe_have("python")) + .or_else(|| cmd_finder.maybe_have("python3")) + .or_else(|| cmd_finder.maybe_have("python2")); + + build.config.nodejs = build + .config + .nodejs + .take() + .map(|p| cmd_finder.must_have(p)) + .or_else(|| cmd_finder.maybe_have("node")) + .or_else(|| cmd_finder.maybe_have("nodejs")); + + build.config.npm = build + .config + .npm + .take() + .map(|p| cmd_finder.must_have(p)) + .or_else(|| cmd_finder.maybe_have("npm")); + + build.config.gdb = build + .config + .gdb + .take() + .map(|p| cmd_finder.must_have(p)) + .or_else(|| cmd_finder.maybe_have("gdb")); + + // We're gonna build some custom C code here and there, host triples + // also build some C++ shims for LLVM so we need a C++ compiler. + for target in &build.targets { + // On emscripten we don't actually need the C compiler to just + // build the target artifacts, only for testing. For the sake + // of easier bot configuration, just skip detection. + if target.contains("emscripten") { + continue; + } + + // We don't use a C compiler on wasm32 + if target.contains("wasm32") { + continue; + } + + if !build.config.dry_run { + cmd_finder.must_have(build.cc(*target)); + if let Some(ar) = build.ar(*target) { + cmd_finder.must_have(ar); + } + } + } + + for host in &build.hosts { + if !build.config.dry_run { + cmd_finder.must_have(build.cxx(*host).unwrap()); + } + } + + if build.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) { + // Externally configured LLVM requires FileCheck to exist + let filecheck = build.llvm_filecheck(build.build); + if !filecheck.starts_with(&build.out) && !filecheck.exists() && build.config.codegen_tests { + panic!("FileCheck executable {:?} does not exist", filecheck); + } + } + + for target in &build.targets { + build + .config + .target_config + .entry(*target) + .or_insert_with(|| Target::from_triple(&target.triple)); + + if target.contains("-none-") || target.contains("nvptx") { + if build.no_std(*target) == Some(false) { + panic!("All the *-none-* and nvptx* targets are no-std targets") + } + } + + // Make sure musl-root is valid + if target.contains("musl") { + // If this is a native target (host is also musl) and no musl-root is given, + // fall back to the system toolchain in /usr before giving up + if build.musl_root(*target).is_none() && build.config.build == *target { + let target = build.config.target_config.entry(*target).or_default(); + target.musl_root = Some("/usr".into()); + } + match build.musl_libdir(*target) { + Some(libdir) => { + if fs::metadata(libdir.join("libc.a")).is_err() { + panic!("couldn't find libc.a in musl libdir: {}", libdir.display()); + } + } + None => panic!( + "when targeting MUSL either the rust.musl-root \ + option or the target.$TARGET.musl-root option must \ + be specified in config.toml" + ), + } + } + + if need_cmake && target.contains("msvc") { + // There are three builds of cmake on windows: MSVC, MinGW, and + // Cygwin. The Cygwin build does not have generators for Visual + // Studio, so detect that here and error. + let out = output(Command::new("cmake").arg("--help")); + if !out.contains("Visual Studio") { + panic!( + " +cmake does not support Visual Studio generators. + +This is likely due to it being an msys/cygwin build of cmake, +rather than the required windows version, built using MinGW +or Visual Studio. + +If you are building under msys2 try installing the mingw-w64-x86_64-cmake +package instead of cmake: + +$ pacman -R cmake && pacman -S mingw-w64-x86_64-cmake +" + ); + } + } + } + + if let Some(ref s) = build.config.ccache { + cmd_finder.must_have(s); + } +} diff --git a/src/bootstrap/setup.rs b/src/bootstrap/setup.rs new file mode 100644 index 000000000..a5a39a5a3 --- /dev/null +++ b/src/bootstrap/setup.rs @@ -0,0 +1,350 @@ +use crate::{t, VERSION}; +use crate::{Config, TargetSelection}; +use std::env::consts::EXE_SUFFIX; +use std::fmt::Write as _; +use std::fs::File; +use std::path::{Path, PathBuf, MAIN_SEPARATOR}; +use std::process::Command; +use std::str::FromStr; +use std::{ + env, fmt, fs, + io::{self, Write}, +}; + +#[derive(Clone, Copy, Eq, PartialEq)] +pub enum Profile { + Compiler, + Codegen, + Library, + Tools, + User, +} + +impl Profile { + fn include_path(&self, src_path: &Path) -> PathBuf { + PathBuf::from(format!("{}/src/bootstrap/defaults/config.{}.toml", src_path.display(), self)) + } + + pub fn all() -> impl Iterator<Item = Self> { + use Profile::*; + // N.B. these are ordered by how they are displayed, not alphabetically + [Library, Compiler, Codegen, Tools, User].iter().copied() + } + + pub fn purpose(&self) -> String { + use Profile::*; + match self { + Library => "Contribute to the standard library", + Compiler => "Contribute to the compiler itself", + Codegen => "Contribute to the compiler, and also modify LLVM or codegen", + Tools => "Contribute to tools which depend on the compiler, but do not modify it directly (e.g. rustdoc, clippy, miri)", + User => "Install Rust from source", + } + .to_string() + } + + pub fn all_for_help(indent: &str) -> String { + let mut out = String::new(); + for choice in Profile::all() { + writeln!(&mut out, "{}{}: {}", indent, choice, choice.purpose()).unwrap(); + } + out + } +} + +impl FromStr for Profile { + type Err = String; + + fn from_str(s: &str) -> Result<Self, Self::Err> { + match s { + "lib" | "library" => Ok(Profile::Library), + "compiler" => Ok(Profile::Compiler), + "llvm" | "codegen" => Ok(Profile::Codegen), + "maintainer" | "user" => Ok(Profile::User), + "tools" | "tool" | "rustdoc" | "clippy" | "miri" | "rustfmt" | "rls" => { + Ok(Profile::Tools) + } + _ => Err(format!("unknown profile: '{}'", s)), + } + } +} + +impl fmt::Display for Profile { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Profile::Compiler => write!(f, "compiler"), + Profile::Codegen => write!(f, "codegen"), + Profile::Library => write!(f, "library"), + Profile::User => write!(f, "user"), + Profile::Tools => write!(f, "tools"), + } + } +} + +pub fn setup(config: &Config, profile: Profile) { + let path = &config.config; + + if path.exists() { + eprintln!( + "error: you asked `x.py` to setup a new config file, but one already exists at `{}`", + path.display() + ); + eprintln!("help: try adding `profile = \"{}\"` at the top of {}", profile, path.display()); + eprintln!( + "note: this will use the configuration in {}", + profile.include_path(&config.src).display() + ); + crate::detail_exit(1); + } + + let settings = format!( + "# Includes one of the default files in src/bootstrap/defaults\n\ + profile = \"{}\"\n\ + changelog-seen = {}\n", + profile, VERSION + ); + t!(fs::write(path, settings)); + + let include_path = profile.include_path(&config.src); + println!("`x.py` will now use the configuration at {}", include_path.display()); + + let build = TargetSelection::from_user(&env!("BUILD_TRIPLE")); + let stage_path = + ["build", build.rustc_target_arg(), "stage1"].join(&MAIN_SEPARATOR.to_string()); + + println!(); + + if !rustup_installed() && profile != Profile::User { + eprintln!("`rustup` is not installed; cannot link `stage1` toolchain"); + } else if stage_dir_exists(&stage_path[..]) { + attempt_toolchain_link(&stage_path[..]); + } + + let suggestions = match profile { + Profile::Codegen | Profile::Compiler => &["check", "build", "test"][..], + Profile::Tools => &[ + "check", + "build", + "test src/test/rustdoc*", + "test src/tools/clippy", + "test src/tools/miri", + "test src/tools/rustfmt", + ], + Profile::Library => &["check", "build", "test library/std", "doc"], + Profile::User => &["dist", "build"], + }; + + println!(); + + t!(install_git_hook_maybe(&config)); + + println!(); + + println!("To get started, try one of the following commands:"); + for cmd in suggestions { + println!("- `x.py {}`", cmd); + } + + if profile != Profile::User { + println!( + "For more suggestions, see https://rustc-dev-guide.rust-lang.org/building/suggested.html" + ); + } +} + +fn rustup_installed() -> bool { + Command::new("rustup") + .arg("--version") + .stdout(std::process::Stdio::null()) + .output() + .map_or(false, |output| output.status.success()) +} + +fn stage_dir_exists(stage_path: &str) -> bool { + match fs::create_dir(&stage_path) { + Ok(_) => true, + Err(_) => Path::new(&stage_path).exists(), + } +} + +fn attempt_toolchain_link(stage_path: &str) { + if toolchain_is_linked() { + return; + } + + if !ensure_stage1_toolchain_placeholder_exists(stage_path) { + eprintln!( + "Failed to create a template for stage 1 toolchain or confirm that it already exists" + ); + return; + } + + if try_link_toolchain(&stage_path) { + println!( + "Added `stage1` rustup toolchain; try `cargo +stage1 build` on a separate rust project to run a newly-built toolchain" + ); + } else { + eprintln!("`rustup` failed to link stage 1 build to `stage1` toolchain"); + eprintln!( + "To manually link stage 1 build to `stage1` toolchain, run:\n + `rustup toolchain link stage1 {}`", + &stage_path + ); + } +} + +fn toolchain_is_linked() -> bool { + match Command::new("rustup") + .args(&["toolchain", "list"]) + .stdout(std::process::Stdio::piped()) + .output() + { + Ok(toolchain_list) => { + if !String::from_utf8_lossy(&toolchain_list.stdout).contains("stage1") { + return false; + } + // The toolchain has already been linked. + println!( + "`stage1` toolchain already linked; not attempting to link `stage1` toolchain" + ); + } + Err(_) => { + // In this case, we don't know if the `stage1` toolchain has been linked; + // but `rustup` failed, so let's not go any further. + println!( + "`rustup` failed to list current toolchains; not attempting to link `stage1` toolchain" + ); + } + } + true +} + +fn try_link_toolchain(stage_path: &str) -> bool { + Command::new("rustup") + .stdout(std::process::Stdio::null()) + .args(&["toolchain", "link", "stage1", &stage_path]) + .output() + .map_or(false, |output| output.status.success()) +} + +fn ensure_stage1_toolchain_placeholder_exists(stage_path: &str) -> bool { + let pathbuf = PathBuf::from(stage_path); + + if fs::create_dir_all(pathbuf.join("lib")).is_err() { + return false; + }; + + let pathbuf = pathbuf.join("bin"); + if fs::create_dir_all(&pathbuf).is_err() { + return false; + }; + + let pathbuf = pathbuf.join(format!("rustc{}", EXE_SUFFIX)); + + if pathbuf.exists() { + return true; + } + + // Take care not to overwrite the file + let result = File::options().append(true).create(true).open(&pathbuf); + if result.is_err() { + return false; + } + + return true; +} + +// Used to get the path for `Subcommand::Setup` +pub fn interactive_path() -> io::Result<Profile> { + fn abbrev_all() -> impl Iterator<Item = ((String, String), Profile)> { + ('a'..) + .zip(1..) + .map(|(letter, number)| (letter.to_string(), number.to_string())) + .zip(Profile::all()) + } + + fn parse_with_abbrev(input: &str) -> Result<Profile, String> { + let input = input.trim().to_lowercase(); + for ((letter, number), profile) in abbrev_all() { + if input == letter || input == number { + return Ok(profile); + } + } + input.parse() + } + + println!("Welcome to the Rust project! What do you want to do with x.py?"); + for ((letter, _), profile) in abbrev_all() { + println!("{}) {}: {}", letter, profile, profile.purpose()); + } + let template = loop { + print!( + "Please choose one ({}): ", + abbrev_all().map(|((l, _), _)| l).collect::<Vec<_>>().join("/") + ); + io::stdout().flush()?; + let mut input = String::new(); + io::stdin().read_line(&mut input)?; + if input.is_empty() { + eprintln!("EOF on stdin, when expecting answer to question. Giving up."); + crate::detail_exit(1); + } + break match parse_with_abbrev(&input) { + Ok(profile) => profile, + Err(err) => { + eprintln!("error: {}", err); + eprintln!("note: press Ctrl+C to exit"); + continue; + } + }; + }; + Ok(template) +} + +// install a git hook to automatically run tidy --bless, if they want +fn install_git_hook_maybe(config: &Config) -> io::Result<()> { + let mut input = String::new(); + println!( + "Rust's CI will automatically fail if it doesn't pass `tidy`, the internal tool for ensuring code quality. +If you'd like, x.py can install a git hook for you that will automatically run `tidy --bless` before +pushing your code to ensure your code is up to par. If you decide later that this behavior is +undesirable, simply delete the `pre-push` file from .git/hooks." + ); + + let should_install = loop { + print!("Would you like to install the git hook?: [y/N] "); + io::stdout().flush()?; + input.clear(); + io::stdin().read_line(&mut input)?; + break match input.trim().to_lowercase().as_str() { + "y" | "yes" => true, + "n" | "no" | "" => false, + _ => { + eprintln!("error: unrecognized option '{}'", input.trim()); + eprintln!("note: press Ctrl+C to exit"); + continue; + } + }; + }; + + if should_install { + let src = config.src.join("src").join("etc").join("pre-push.sh"); + let git = + t!(config.git().args(&["rev-parse", "--git-common-dir"]).output().map(|output| { + assert!(output.status.success(), "failed to run `git`"); + PathBuf::from(t!(String::from_utf8(output.stdout)).trim()) + })); + let dst = git.join("hooks").join("pre-push"); + match fs::hard_link(src, &dst) { + Err(e) => eprintln!( + "error: could not create hook {}: do you already have the git hook installed?\n{}", + dst.display(), + e + ), + Ok(_) => println!("Linked `src/etc/pre-push.sh` to `.git/hooks/pre-push`"), + }; + } else { + println!("Ok, skipping installation!"); + } + Ok(()) +} diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs new file mode 100644 index 000000000..7b0c029c1 --- /dev/null +++ b/src/bootstrap/tarball.rs @@ -0,0 +1,374 @@ +use std::{ + path::{Path, PathBuf}, + process::Command, +}; + +use crate::builder::Builder; +use crate::util::t; + +#[derive(Copy, Clone)] +pub(crate) enum OverlayKind { + Rust, + LLVM, + Cargo, + Clippy, + Miri, + Rustfmt, + RustDemangler, + RLS, + RustAnalyzer, +} + +impl OverlayKind { + fn legal_and_readme(&self) -> &[&str] { + match self { + OverlayKind::Rust => &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"], + OverlayKind::LLVM => { + &["src/llvm-project/llvm/LICENSE.TXT", "src/llvm-project/llvm/README.txt"] + } + OverlayKind::Cargo => &[ + "src/tools/cargo/README.md", + "src/tools/cargo/LICENSE-MIT", + "src/tools/cargo/LICENSE-APACHE", + "src/tools/cargo/LICENSE-THIRD-PARTY", + ], + OverlayKind::Clippy => &[ + "src/tools/clippy/README.md", + "src/tools/clippy/LICENSE-APACHE", + "src/tools/clippy/LICENSE-MIT", + ], + OverlayKind::Miri => &[ + "src/tools/miri/README.md", + "src/tools/miri/LICENSE-APACHE", + "src/tools/miri/LICENSE-MIT", + ], + OverlayKind::Rustfmt => &[ + "src/tools/rustfmt/README.md", + "src/tools/rustfmt/LICENSE-APACHE", + "src/tools/rustfmt/LICENSE-MIT", + ], + OverlayKind::RustDemangler => { + &["src/tools/rust-demangler/README.md", "LICENSE-APACHE", "LICENSE-MIT"] + } + OverlayKind::RLS => &[ + "src/tools/rls/README.md", + "src/tools/rls/LICENSE-APACHE", + "src/tools/rls/LICENSE-MIT", + ], + OverlayKind::RustAnalyzer => &[ + "src/tools/rust-analyzer/README.md", + "src/tools/rust-analyzer/LICENSE-APACHE", + "src/tools/rust-analyzer/LICENSE-MIT", + ], + } + } + + fn version(&self, builder: &Builder<'_>) -> String { + match self { + OverlayKind::Rust => builder.rust_version(), + OverlayKind::LLVM => builder.rust_version(), + OverlayKind::RustDemangler => builder.release_num("rust-demangler"), + OverlayKind::Cargo => { + builder.cargo_info.version(builder, &builder.release_num("cargo")) + } + OverlayKind::Clippy => { + builder.clippy_info.version(builder, &builder.release_num("clippy")) + } + OverlayKind::Miri => builder.miri_info.version(builder, &builder.release_num("miri")), + OverlayKind::Rustfmt => { + builder.rustfmt_info.version(builder, &builder.release_num("rustfmt")) + } + OverlayKind::RLS => builder.rls_info.version(builder, &builder.release_num("rls")), + OverlayKind::RustAnalyzer => builder + .rust_analyzer_info + .version(builder, &builder.release_num("rust-analyzer/crates/rust-analyzer")), + } + } +} + +pub(crate) struct Tarball<'a> { + builder: &'a Builder<'a>, + + pkgname: String, + component: String, + target: Option<String>, + product_name: String, + overlay: OverlayKind, + + temp_dir: PathBuf, + image_dir: PathBuf, + overlay_dir: PathBuf, + bulk_dirs: Vec<PathBuf>, + + include_target_in_component_name: bool, + is_preview: bool, + permit_symlinks: bool, +} + +impl<'a> Tarball<'a> { + pub(crate) fn new(builder: &'a Builder<'a>, component: &str, target: &str) -> Self { + Self::new_inner(builder, component, Some(target.into())) + } + + pub(crate) fn new_targetless(builder: &'a Builder<'a>, component: &str) -> Self { + Self::new_inner(builder, component, None) + } + + fn new_inner(builder: &'a Builder<'a>, component: &str, target: Option<String>) -> Self { + let pkgname = crate::dist::pkgname(builder, component); + + let mut temp_dir = builder.out.join("tmp").join("tarball").join(component); + if let Some(target) = &target { + temp_dir = temp_dir.join(target); + } + let _ = std::fs::remove_dir_all(&temp_dir); + + let image_dir = temp_dir.join("image"); + let overlay_dir = temp_dir.join("overlay"); + + Self { + builder, + + pkgname, + component: component.into(), + target, + product_name: "Rust".into(), + overlay: OverlayKind::Rust, + + temp_dir, + image_dir, + overlay_dir, + bulk_dirs: Vec::new(), + + include_target_in_component_name: false, + is_preview: false, + permit_symlinks: false, + } + } + + pub(crate) fn set_overlay(&mut self, overlay: OverlayKind) { + self.overlay = overlay; + } + + pub(crate) fn set_product_name(&mut self, name: &str) { + self.product_name = name.into(); + } + + pub(crate) fn include_target_in_component_name(&mut self, include: bool) { + self.include_target_in_component_name = include; + } + + pub(crate) fn is_preview(&mut self, is: bool) { + self.is_preview = is; + } + + pub(crate) fn permit_symlinks(&mut self, flag: bool) { + self.permit_symlinks = flag; + } + + pub(crate) fn image_dir(&self) -> &Path { + t!(std::fs::create_dir_all(&self.image_dir)); + &self.image_dir + } + + pub(crate) fn add_file(&self, src: impl AsRef<Path>, destdir: impl AsRef<Path>, perms: u32) { + // create_dir_all fails to create `foo/bar/.`, so when the destination is "." this simply + // uses the base directory as the destination directory. + let destdir = if destdir.as_ref() == Path::new(".") { + self.image_dir.clone() + } else { + self.image_dir.join(destdir.as_ref()) + }; + + t!(std::fs::create_dir_all(&destdir)); + self.builder.install(src.as_ref(), &destdir, perms); + } + + pub(crate) fn add_renamed_file( + &self, + src: impl AsRef<Path>, + destdir: impl AsRef<Path>, + new_name: &str, + ) { + let destdir = self.image_dir.join(destdir.as_ref()); + t!(std::fs::create_dir_all(&destdir)); + self.builder.copy(src.as_ref(), &destdir.join(new_name)); + } + + pub(crate) fn add_legal_and_readme_to(&self, destdir: impl AsRef<Path>) { + for file in self.overlay.legal_and_readme() { + self.add_file(self.builder.src.join(file), destdir.as_ref(), 0o644); + } + } + + pub(crate) fn add_dir(&self, src: impl AsRef<Path>, dest: impl AsRef<Path>) { + let dest = self.image_dir.join(dest.as_ref()); + + t!(std::fs::create_dir_all(&dest)); + self.builder.cp_r(src.as_ref(), &dest); + } + + pub(crate) fn add_bulk_dir(&mut self, src: impl AsRef<Path>, dest: impl AsRef<Path>) { + self.bulk_dirs.push(dest.as_ref().to_path_buf()); + self.add_dir(src, dest); + } + + pub(crate) fn generate(self) -> GeneratedTarball { + let mut component_name = self.component.clone(); + if self.is_preview { + component_name.push_str("-preview"); + } + if self.include_target_in_component_name { + component_name.push('-'); + component_name.push_str( + &self + .target + .as_ref() + .expect("include_target_in_component_name used in a targetless tarball"), + ); + } + + self.run(|this, cmd| { + cmd.arg("generate") + .arg("--image-dir") + .arg(&this.image_dir) + .arg(format!("--component-name={}", &component_name)); + + if let Some((dir, dirs)) = this.bulk_dirs.split_first() { + let mut arg = dir.as_os_str().to_os_string(); + for dir in dirs { + arg.push(","); + arg.push(dir); + } + cmd.arg("--bulk-dirs").arg(&arg); + } + + this.non_bare_args(cmd); + }) + } + + pub(crate) fn combine(self, tarballs: &[GeneratedTarball]) -> GeneratedTarball { + let mut input_tarballs = tarballs[0].path.as_os_str().to_os_string(); + for tarball in &tarballs[1..] { + input_tarballs.push(","); + input_tarballs.push(&tarball.path); + } + + self.run(|this, cmd| { + cmd.arg("combine").arg("--input-tarballs").arg(input_tarballs); + this.non_bare_args(cmd); + }) + } + + pub(crate) fn bare(self) -> GeneratedTarball { + // Bare tarballs should have the top level directory match the package + // name, not "image". We rename the image directory just before passing + // into rust-installer. + let dest = self.temp_dir.join(self.package_name()); + t!(std::fs::rename(&self.image_dir, &dest)); + + self.run(|this, cmd| { + let distdir = crate::dist::distdir(this.builder); + t!(std::fs::create_dir_all(&distdir)); + cmd.arg("tarball") + .arg("--input") + .arg(&dest) + .arg("--output") + .arg(distdir.join(this.package_name())); + }) + } + + fn package_name(&self) -> String { + if let Some(target) = &self.target { + format!("{}-{}", self.pkgname, target) + } else { + self.pkgname.clone() + } + } + + fn non_bare_args(&self, cmd: &mut Command) { + cmd.arg("--rel-manifest-dir=rustlib") + .arg("--legacy-manifest-dirs=rustlib,cargo") + .arg(format!("--product-name={}", self.product_name)) + .arg(format!("--success-message={} installed.", self.component)) + .arg(format!("--package-name={}", self.package_name())) + .arg("--non-installed-overlay") + .arg(&self.overlay_dir) + .arg("--output-dir") + .arg(crate::dist::distdir(self.builder)); + } + + fn run(self, build_cli: impl FnOnce(&Tarball<'a>, &mut Command)) -> GeneratedTarball { + t!(std::fs::create_dir_all(&self.overlay_dir)); + self.builder.create(&self.overlay_dir.join("version"), &self.overlay.version(self.builder)); + if let Some(sha) = self.builder.rust_sha() { + self.builder.create(&self.overlay_dir.join("git-commit-hash"), &sha); + } + for file in self.overlay.legal_and_readme() { + self.builder.install(&self.builder.src.join(file), &self.overlay_dir, 0o644); + } + + let mut cmd = self.builder.tool_cmd(crate::tool::Tool::RustInstaller); + + let package_name = self.package_name(); + self.builder.info(&format!("Dist {}", package_name)); + let _time = crate::util::timeit(self.builder); + + build_cli(&self, &mut cmd); + cmd.arg("--work-dir").arg(&self.temp_dir); + if let Some(formats) = &self.builder.config.dist_compression_formats { + assert!(!formats.is_empty(), "dist.compression-formats can't be empty"); + cmd.arg("--compression-formats").arg(formats.join(",")); + } + self.builder.run(&mut cmd); + + // Ensure there are no symbolic links in the tarball. In particular, + // rustup-toolchain-install-master and most versions of Windows can't handle symbolic links. + let decompressed_output = self.temp_dir.join(&package_name); + if !self.builder.config.dry_run && !self.permit_symlinks { + for entry in walkdir::WalkDir::new(&decompressed_output) { + let entry = t!(entry); + if entry.path_is_symlink() { + panic!("generated a symlink in a tarball: {}", entry.path().display()); + } + } + } + + // Use either the first compression format defined, or "gz" as the default. + let ext = self + .builder + .config + .dist_compression_formats + .as_ref() + .and_then(|formats| formats.get(0)) + .map(|s| s.as_str()) + .unwrap_or("gz"); + + GeneratedTarball { + path: crate::dist::distdir(self.builder).join(format!("{}.tar.{}", package_name, ext)), + decompressed_output, + work: self.temp_dir, + } + } +} + +#[derive(Debug, Clone)] +pub struct GeneratedTarball { + path: PathBuf, + decompressed_output: PathBuf, + work: PathBuf, +} + +impl GeneratedTarball { + pub(crate) fn tarball(&self) -> &Path { + &self.path + } + + pub(crate) fn decompressed_output(&self) -> &Path { + &self.decompressed_output + } + + pub(crate) fn work_dir(&self) -> &Path { + &self.work + } +} diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs new file mode 100644 index 000000000..c0fa8c9ac --- /dev/null +++ b/src/bootstrap/test.rs @@ -0,0 +1,2542 @@ +//! Implementation of the test-related targets of the build system. +//! +//! This file implements the various regression test suites that we execute on +//! our CI. + +use std::env; +use std::ffi::OsString; +use std::fmt; +use std::fs; +use std::iter; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +use crate::builder::{Builder, Compiler, Kind, RunConfig, ShouldRun, Step}; +use crate::cache::Interned; +use crate::compile; +use crate::config::TargetSelection; +use crate::dist; +use crate::flags::Subcommand; +use crate::native; +use crate::tool::{self, SourceType, Tool}; +use crate::toolstate::ToolState; +use crate::util::{self, add_link_lib_path, dylib_path, dylib_path_var, output, t}; +use crate::{envify, CLang, DocTests, GitRepo, Mode}; + +const ADB_TEST_DIR: &str = "/data/tmp/work"; + +/// The two modes of the test runner; tests or benchmarks. +#[derive(Debug, PartialEq, Eq, Hash, Copy, Clone, PartialOrd, Ord)] +pub enum TestKind { + /// Run `cargo test`. + Test, + /// Run `cargo bench`. + Bench, +} + +impl From<Kind> for TestKind { + fn from(kind: Kind) -> Self { + match kind { + Kind::Test => TestKind::Test, + Kind::Bench => TestKind::Bench, + _ => panic!("unexpected kind in crate: {:?}", kind), + } + } +} + +impl TestKind { + // Return the cargo subcommand for this test kind + fn subcommand(self) -> &'static str { + match self { + TestKind::Test => "test", + TestKind::Bench => "bench", + } + } +} + +impl fmt::Display for TestKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match *self { + TestKind::Test => "Testing", + TestKind::Bench => "Benchmarking", + }) + } +} + +fn try_run(builder: &Builder<'_>, cmd: &mut Command) -> bool { + if !builder.fail_fast { + if !builder.try_run(cmd) { + let mut failures = builder.delayed_failures.borrow_mut(); + failures.push(format!("{:?}", cmd)); + return false; + } + } else { + builder.run(cmd); + } + true +} + +fn try_run_quiet(builder: &Builder<'_>, cmd: &mut Command) -> bool { + if !builder.fail_fast { + if !builder.try_run_quiet(cmd) { + let mut failures = builder.delayed_failures.borrow_mut(); + failures.push(format!("{:?}", cmd)); + return false; + } + } else { + builder.run_quiet(cmd); + } + true +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Linkcheck { + host: TargetSelection, +} + +impl Step for Linkcheck { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + /// Runs the `linkchecker` tool as compiled in `stage` by the `host` compiler. + /// + /// This tool in `src/tools` will verify the validity of all our links in the + /// documentation to ensure we don't have a bunch of dead ones. + fn run(self, builder: &Builder<'_>) { + let host = self.host; + let hosts = &builder.hosts; + let targets = &builder.targets; + + // if we have different hosts and targets, some things may be built for + // the host (e.g. rustc) and others for the target (e.g. std). The + // documentation built for each will contain broken links to + // docs built for the other platform (e.g. rustc linking to cargo) + if (hosts != targets) && !hosts.is_empty() && !targets.is_empty() { + panic!( + "Linkcheck currently does not support builds with different hosts and targets. +You can skip linkcheck with --exclude src/tools/linkchecker" + ); + } + + builder.info(&format!("Linkcheck ({})", host)); + + // Test the linkchecker itself. + let bootstrap_host = builder.config.build; + let compiler = builder.compiler(0, bootstrap_host); + let cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolBootstrap, + bootstrap_host, + "test", + "src/tools/linkchecker", + SourceType::InTree, + &[], + ); + try_run(builder, &mut cargo.into()); + + // Build all the default documentation. + builder.default_doc(&[]); + + // Run the linkchecker. + let _time = util::timeit(&builder); + try_run( + builder, + builder.tool_cmd(Tool::Linkchecker).arg(builder.out.join(host.triple).join("doc")), + ); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + let run = run.path("src/tools/linkchecker"); + run.default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Linkcheck { host: run.target }); + } +} + +fn check_if_tidy_is_installed() -> bool { + Command::new("tidy") + .arg("--version") + .stdout(Stdio::null()) + .status() + .map_or(false, |status| status.success()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct HtmlCheck { + target: TargetSelection, +} + +impl Step for HtmlCheck { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let run = run.path("src/tools/html-checker"); + run.lazy_default_condition(Box::new(check_if_tidy_is_installed)) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(HtmlCheck { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + if !check_if_tidy_is_installed() { + eprintln!("not running HTML-check tool because `tidy` is missing"); + eprintln!( + "Note that `tidy` is not the in-tree `src/tools/tidy` but needs to be installed" + ); + panic!("Cannot run html-check tests"); + } + // Ensure that a few different kinds of documentation are available. + builder.default_doc(&[]); + builder.ensure(crate::doc::Rustc { target: self.target, stage: builder.top_stage }); + + try_run(builder, builder.tool_cmd(Tool::HtmlChecker).arg(builder.doc_out(self.target))); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Cargotest { + stage: u32, + host: TargetSelection, +} + +impl Step for Cargotest { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/cargotest") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Cargotest { stage: run.builder.top_stage, host: run.target }); + } + + /// Runs the `cargotest` tool as compiled in `stage` by the `host` compiler. + /// + /// This tool in `src/tools` will check out a few Rust projects and run `cargo + /// test` to ensure that we don't regress the test suites there. + fn run(self, builder: &Builder<'_>) { + let compiler = builder.compiler(self.stage, self.host); + builder.ensure(compile::Rustc::new(compiler, compiler.host)); + let cargo = builder.ensure(tool::Cargo { compiler, target: compiler.host }); + + // Note that this is a short, cryptic, and not scoped directory name. This + // is currently to minimize the length of path on Windows where we otherwise + // quickly run into path name limit constraints. + let out_dir = builder.out.join("ct"); + t!(fs::create_dir_all(&out_dir)); + + let _time = util::timeit(&builder); + let mut cmd = builder.tool_cmd(Tool::CargoTest); + try_run( + builder, + cmd.arg(&cargo) + .arg(&out_dir) + .args(builder.config.cmd.test_args()) + .env("RUSTC", builder.rustc(compiler)) + .env("RUSTDOC", builder.rustdoc(compiler)), + ); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Cargo { + stage: u32, + host: TargetSelection, +} + +impl Step for Cargo { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/cargo") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Cargo { stage: run.builder.top_stage, host: run.target }); + } + + /// Runs `cargo test` for `cargo` packaged with Rust. + fn run(self, builder: &Builder<'_>) { + let compiler = builder.compiler(self.stage, self.host); + + builder.ensure(tool::Cargo { compiler, target: self.host }); + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + self.host, + "test", + "src/tools/cargo", + SourceType::Submodule, + &[], + ); + + if !builder.fail_fast { + cargo.arg("--no-fail-fast"); + } + cargo.arg("--").args(builder.config.cmd.test_args()); + + // Don't run cross-compile tests, we may not have cross-compiled libstd libs + // available. + cargo.env("CFG_DISABLE_CROSS_TESTS", "1"); + // Forcibly disable tests using nightly features since any changes to + // those features won't be able to land. + cargo.env("CARGO_TEST_DISABLE_NIGHTLY", "1"); + + cargo.env("PATH", &path_for_cargo(builder, compiler)); + + try_run(builder, &mut cargo.into()); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Rls { + stage: u32, + host: TargetSelection, +} + +impl Step for Rls { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rls") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Rls { stage: run.builder.top_stage, host: run.target }); + } + + /// Runs `cargo test` for the rls. + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let host = self.host; + let compiler = builder.compiler(stage, host); + + let build_result = + builder.ensure(tool::Rls { compiler, target: self.host, extra_features: Vec::new() }); + if build_result.is_none() { + eprintln!("failed to test rls: could not build"); + return; + } + + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + host, + "test", + "src/tools/rls", + SourceType::Submodule, + &[], + ); + + cargo.add_rustc_lib_path(builder, compiler); + cargo.arg("--").args(builder.config.cmd.test_args()); + + if try_run(builder, &mut cargo.into()) { + builder.save_toolstate("rls", ToolState::TestPass); + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct RustAnalyzer { + stage: u32, + host: TargetSelection, +} + +impl Step for RustAnalyzer { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rust-analyzer") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Self { stage: run.builder.top_stage, host: run.target }); + } + + /// Runs `cargo test` for rust-analyzer + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let host = self.host; + let compiler = builder.compiler(stage, host); + + builder.ensure(tool::RustAnalyzer { compiler, target: self.host }).expect("in-tree tool"); + + let workspace_path = "src/tools/rust-analyzer"; + // until the whole RA test suite runs on `i686`, we only run + // `proc-macro-srv` tests + let crate_path = "src/tools/rust-analyzer/crates/proc-macro-srv"; + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolStd, + host, + "test", + crate_path, + SourceType::InTree, + &["sysroot-abi".to_owned()], + ); + + let dir = builder.src.join(workspace_path); + // needed by rust-analyzer to find its own text fixtures, cf. + // https://github.com/rust-analyzer/expect-test/issues/33 + cargo.env("CARGO_WORKSPACE_DIR", &dir); + + // RA's test suite tries to write to the source directory, that can't + // work in Rust CI + cargo.env("SKIP_SLOW_TESTS", "1"); + + cargo.add_rustc_lib_path(builder, compiler); + cargo.arg("--").args(builder.config.cmd.test_args()); + + builder.run(&mut cargo.into()); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Rustfmt { + stage: u32, + host: TargetSelection, +} + +impl Step for Rustfmt { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rustfmt") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Rustfmt { stage: run.builder.top_stage, host: run.target }); + } + + /// Runs `cargo test` for rustfmt. + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let host = self.host; + let compiler = builder.compiler(stage, host); + + builder + .ensure(tool::Rustfmt { compiler, target: self.host, extra_features: Vec::new() }) + .expect("in-tree tool"); + + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + host, + "test", + "src/tools/rustfmt", + SourceType::InTree, + &[], + ); + + let dir = testdir(builder, compiler.host); + t!(fs::create_dir_all(&dir)); + cargo.env("RUSTFMT_TEST_DIR", dir); + + cargo.add_rustc_lib_path(builder, compiler); + + builder.run(&mut cargo.into()); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct RustDemangler { + stage: u32, + host: TargetSelection, +} + +impl Step for RustDemangler { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rust-demangler") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustDemangler { stage: run.builder.top_stage, host: run.target }); + } + + /// Runs `cargo test` for rust-demangler. + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let host = self.host; + let compiler = builder.compiler(stage, host); + + let rust_demangler = builder + .ensure(tool::RustDemangler { compiler, target: self.host, extra_features: Vec::new() }) + .expect("in-tree tool"); + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + host, + "test", + "src/tools/rust-demangler", + SourceType::InTree, + &[], + ); + + let dir = testdir(builder, compiler.host); + t!(fs::create_dir_all(&dir)); + + cargo.env("RUST_DEMANGLER_DRIVER_PATH", rust_demangler); + + cargo.arg("--").args(builder.config.cmd.test_args()); + + cargo.add_rustc_lib_path(builder, compiler); + + builder.run(&mut cargo.into()); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Miri { + stage: u32, + host: TargetSelection, +} + +impl Step for Miri { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/miri") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Miri { stage: run.builder.top_stage, host: run.target }); + } + + /// Runs `cargo test` for miri. + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let host = self.host; + let compiler = builder.compiler(stage, host); + // We need the stdlib for the *next* stage, as it was built with this compiler that also built Miri. + // Except if we are at stage 2, the bootstrap loop is complete and we can stick with our current stage. + let compiler_std = builder.compiler(if stage < 2 { stage + 1 } else { stage }, host); + + let miri = + builder.ensure(tool::Miri { compiler, target: self.host, extra_features: Vec::new() }); + let cargo_miri = builder.ensure(tool::CargoMiri { + compiler, + target: self.host, + extra_features: Vec::new(), + }); + // The stdlib we need might be at a different stage. And just asking for the + // sysroot does not seem to populate it, so we do that first. + builder.ensure(compile::Std::new(compiler_std, host)); + let sysroot = builder.sysroot(compiler_std); + if let (Some(miri), Some(_cargo_miri)) = (miri, cargo_miri) { + let mut cargo = + builder.cargo(compiler, Mode::ToolRustc, SourceType::Submodule, host, "install"); + cargo.arg("xargo"); + // Configure `cargo install` path. cargo adds a `bin/`. + cargo.env("CARGO_INSTALL_ROOT", &builder.out); + + let mut cargo = Command::from(cargo); + if !try_run(builder, &mut cargo) { + return; + } + + // # Run `cargo miri setup`. + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + host, + "run", + "src/tools/miri/cargo-miri", + SourceType::Submodule, + &[], + ); + cargo.add_rustc_lib_path(builder, compiler); + cargo.arg("--").arg("miri").arg("setup"); + + // Tell `cargo miri setup` where to find the sources. + cargo.env("XARGO_RUST_SRC", builder.src.join("library")); + // Tell it where to find Miri. + cargo.env("MIRI", &miri); + // Debug things. + cargo.env("RUST_BACKTRACE", "1"); + // Let cargo-miri know where xargo ended up. + cargo.env("XARGO_CHECK", builder.out.join("bin").join("xargo-check")); + + let mut cargo = Command::from(cargo); + if !try_run(builder, &mut cargo) { + return; + } + + // # Determine where Miri put its sysroot. + // To this end, we run `cargo miri setup --print-sysroot` and capture the output. + // (We do this separately from the above so that when the setup actually + // happens we get some output.) + // We re-use the `cargo` from above. + cargo.arg("--print-sysroot"); + + // FIXME: Is there a way in which we can re-use the usual `run` helpers? + let miri_sysroot = if builder.config.dry_run { + String::new() + } else { + builder.verbose(&format!("running: {:?}", cargo)); + let out = cargo + .output() + .expect("We already ran `cargo miri setup` before and that worked"); + assert!(out.status.success(), "`cargo miri setup` returned with non-0 exit code"); + // Output is "<sysroot>\n". + let stdout = String::from_utf8(out.stdout) + .expect("`cargo miri setup` stdout is not valid UTF-8"); + let sysroot = stdout.trim_end(); + builder.verbose(&format!("`cargo miri setup --print-sysroot` said: {:?}", sysroot)); + sysroot.to_owned() + }; + + // # Run `cargo test`. + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + host, + "test", + "src/tools/miri", + SourceType::Submodule, + &[], + ); + cargo.add_rustc_lib_path(builder, compiler); + + // miri tests need to know about the stage sysroot + cargo.env("MIRI_SYSROOT", miri_sysroot); + cargo.env("MIRI_HOST_SYSROOT", sysroot); + cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler)); + cargo.env("MIRI", miri); + + cargo.arg("--").args(builder.config.cmd.test_args()); + + let mut cargo = Command::from(cargo); + if !try_run(builder, &mut cargo) { + return; + } + + // # Done! + builder.save_toolstate("miri", ToolState::TestPass); + } else { + eprintln!("failed to test miri: could not build"); + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct CompiletestTest { + host: TargetSelection, +} + +impl Step for CompiletestTest { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/compiletest") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(CompiletestTest { host: run.target }); + } + + /// Runs `cargo test` for compiletest. + fn run(self, builder: &Builder<'_>) { + let host = self.host; + let compiler = builder.compiler(0, host); + + // We need `ToolStd` for the locally-built sysroot because + // compiletest uses unstable features of the `test` crate. + builder.ensure(compile::Std::new(compiler, host)); + let cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolStd, + host, + "test", + "src/tools/compiletest", + SourceType::InTree, + &[], + ); + + try_run(builder, &mut cargo.into()); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Clippy { + stage: u32, + host: TargetSelection, +} + +impl Step for Clippy { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/clippy") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Clippy { stage: run.builder.top_stage, host: run.target }); + } + + /// Runs `cargo test` for clippy. + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let host = self.host; + let compiler = builder.compiler(stage, host); + + builder + .ensure(tool::Clippy { compiler, target: self.host, extra_features: Vec::new() }) + .expect("in-tree tool"); + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + host, + "test", + "src/tools/clippy", + SourceType::InTree, + &[], + ); + + cargo.env("RUSTC_TEST_SUITE", builder.rustc(compiler)); + cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler)); + let host_libs = builder.stage_out(compiler, Mode::ToolRustc).join(builder.cargo_dir()); + cargo.env("HOST_LIBS", host_libs); + + cargo.arg("--").args(builder.config.cmd.test_args()); + + cargo.add_rustc_lib_path(builder, compiler); + + if builder.try_run(&mut cargo.into()) { + // The tests succeeded; nothing to do. + return; + } + + if !builder.config.cmd.bless() { + crate::detail_exit(1); + } + + let mut cargo = builder.cargo(compiler, Mode::ToolRustc, SourceType::InTree, host, "run"); + cargo.arg("-p").arg("clippy_dev"); + // clippy_dev gets confused if it can't find `clippy/Cargo.toml` + cargo.current_dir(&builder.src.join("src").join("tools").join("clippy")); + if builder.config.rust_optimize { + cargo.env("PROFILE", "release"); + } else { + cargo.env("PROFILE", "debug"); + } + cargo.arg("--"); + cargo.arg("bless"); + builder.run(&mut cargo.into()); + } +} + +fn path_for_cargo(builder: &Builder<'_>, compiler: Compiler) -> OsString { + // Configure PATH to find the right rustc. NB. we have to use PATH + // and not RUSTC because the Cargo test suite has tests that will + // fail if rustc is not spelled `rustc`. + let path = builder.sysroot(compiler).join("bin"); + let old_path = env::var_os("PATH").unwrap_or_default(); + env::join_paths(iter::once(path).chain(env::split_paths(&old_path))).expect("") +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustdocTheme { + pub compiler: Compiler, +} + +impl Step for RustdocTheme { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rustdoc-themes") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + + run.builder.ensure(RustdocTheme { compiler }); + } + + fn run(self, builder: &Builder<'_>) { + let rustdoc = builder.bootstrap_out.join("rustdoc"); + let mut cmd = builder.tool_cmd(Tool::RustdocTheme); + cmd.arg(rustdoc.to_str().unwrap()) + .arg(builder.src.join("src/librustdoc/html/static/css/themes").to_str().unwrap()) + .env("RUSTC_STAGE", self.compiler.stage.to_string()) + .env("RUSTC_SYSROOT", builder.sysroot(self.compiler)) + .env("RUSTDOC_LIBDIR", builder.sysroot_libdir(self.compiler, self.compiler.host)) + .env("CFG_RELEASE_CHANNEL", &builder.config.channel) + .env("RUSTDOC_REAL", builder.rustdoc(self.compiler)) + .env("RUSTC_BOOTSTRAP", "1"); + if let Some(linker) = builder.linker(self.compiler.host) { + cmd.env("RUSTDOC_LINKER", linker); + } + if builder.is_fuse_ld_lld(self.compiler.host) { + cmd.env("RUSTDOC_FUSE_LD_LLD", "1"); + } + try_run(builder, &mut cmd); + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustdocJSStd { + pub target: TargetSelection, +} + +impl Step for RustdocJSStd { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.suite_path("src/test/rustdoc-js-std") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustdocJSStd { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + if let Some(ref nodejs) = builder.config.nodejs { + let mut command = Command::new(nodejs); + command + .arg(builder.src.join("src/tools/rustdoc-js/tester.js")) + .arg("--crate-name") + .arg("std") + .arg("--resource-suffix") + .arg(&builder.version) + .arg("--doc-folder") + .arg(builder.doc_out(self.target)) + .arg("--test-folder") + .arg(builder.src.join("src/test/rustdoc-js-std")); + for path in &builder.paths { + if let Some(p) = + util::is_valid_test_suite_arg(path, "src/test/rustdoc-js-std", builder) + { + if !p.ends_with(".js") { + eprintln!("A non-js file was given: `{}`", path.display()); + panic!("Cannot run rustdoc-js-std tests"); + } + command.arg("--test-file").arg(path); + } + } + builder.ensure(crate::doc::Std { target: self.target, stage: builder.top_stage }); + builder.run(&mut command); + } else { + builder.info("No nodejs found, skipping \"src/test/rustdoc-js-std\" tests"); + } + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustdocJSNotStd { + pub target: TargetSelection, + pub compiler: Compiler, +} + +impl Step for RustdocJSNotStd { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.suite_path("src/test/rustdoc-js") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + run.builder.ensure(RustdocJSNotStd { target: run.target, compiler }); + } + + fn run(self, builder: &Builder<'_>) { + if builder.config.nodejs.is_some() { + builder.ensure(Compiletest { + compiler: self.compiler, + target: self.target, + mode: "js-doc-test", + suite: "rustdoc-js", + path: "src/test/rustdoc-js", + compare_mode: None, + }); + } else { + builder.info("No nodejs found, skipping \"src/test/rustdoc-js\" tests"); + } + } +} + +fn get_browser_ui_test_version_inner(npm: &Path, global: bool) -> Option<String> { + let mut command = Command::new(&npm); + command.arg("list").arg("--parseable").arg("--long").arg("--depth=0"); + if global { + command.arg("--global"); + } + let lines = command + .output() + .map(|output| String::from_utf8_lossy(&output.stdout).into_owned()) + .unwrap_or(String::new()); + lines.lines().find_map(|l| l.split(":browser-ui-test@").skip(1).next()).map(|v| v.to_owned()) +} + +fn get_browser_ui_test_version(npm: &Path) -> Option<String> { + get_browser_ui_test_version_inner(npm, false) + .or_else(|| get_browser_ui_test_version_inner(npm, true)) +} + +fn compare_browser_ui_test_version(installed_version: &str, src: &Path) { + match fs::read_to_string( + src.join("src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version"), + ) { + Ok(v) => { + if v.trim() != installed_version { + eprintln!( + "⚠️ Installed version of browser-ui-test (`{}`) is different than the \ + one used in the CI (`{}`)", + installed_version, v + ); + } + } + Err(e) => eprintln!("Couldn't find the CI browser-ui-test version: {:?}", e), + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustdocGUI { + pub target: TargetSelection, + pub compiler: Compiler, +} + +impl Step for RustdocGUI { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + let run = run.suite_path("src/test/rustdoc-gui"); + run.lazy_default_condition(Box::new(move || { + builder.config.nodejs.is_some() + && builder + .config + .npm + .as_ref() + .map(|p| get_browser_ui_test_version(p).is_some()) + .unwrap_or(false) + })) + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + run.builder.ensure(RustdocGUI { target: run.target, compiler }); + } + + fn run(self, builder: &Builder<'_>) { + let nodejs = builder.config.nodejs.as_ref().expect("nodejs isn't available"); + let npm = builder.config.npm.as_ref().expect("npm isn't available"); + + builder.ensure(compile::Std::new(self.compiler, self.target)); + + // The goal here is to check if the necessary packages are installed, and if not, we + // panic. + match get_browser_ui_test_version(&npm) { + Some(version) => { + // We also check the version currently used in CI and emit a warning if it's not the + // same one. + compare_browser_ui_test_version(&version, &builder.build.src); + } + None => { + eprintln!( + "error: rustdoc-gui test suite cannot be run because npm `browser-ui-test` \ + dependency is missing", + ); + eprintln!( + "If you want to install the `{0}` dependency, run `npm install {0}`", + "browser-ui-test", + ); + panic!("Cannot run rustdoc-gui tests"); + } + } + + let out_dir = builder.test_out(self.target).join("rustdoc-gui"); + + // We remove existing folder to be sure there won't be artifacts remaining. + builder.clear_if_dirty(&out_dir, &builder.rustdoc(self.compiler)); + + let src_path = builder.build.src.join("src/test/rustdoc-gui/src"); + // We generate docs for the libraries present in the rustdoc-gui's src folder. + for entry in src_path.read_dir().expect("read_dir call failed") { + if let Ok(entry) = entry { + let path = entry.path(); + + if !path.is_dir() { + continue; + } + + let mut cargo = Command::new(&builder.initial_cargo); + cargo + .arg("doc") + .arg("--target-dir") + .arg(&out_dir) + .env("RUSTC_BOOTSTRAP", "1") + .env("RUSTDOC", builder.rustdoc(self.compiler)) + .env("RUSTC", builder.rustc(self.compiler)) + .current_dir(path); + // FIXME: implement a `// compile-flags` command or similar + // instead of hard-coding this test + if entry.file_name() == "link_to_definition" { + cargo.env("RUSTDOCFLAGS", "-Zunstable-options --generate-link-to-definition"); + } + builder.run(&mut cargo); + } + } + + // We now run GUI tests. + let mut command = Command::new(&nodejs); + command + .arg(builder.build.src.join("src/tools/rustdoc-gui/tester.js")) + .arg("--jobs") + .arg(&builder.jobs().to_string()) + .arg("--doc-folder") + .arg(out_dir.join("doc")) + .arg("--tests-folder") + .arg(builder.build.src.join("src/test/rustdoc-gui")); + for path in &builder.paths { + if let Some(p) = util::is_valid_test_suite_arg(path, "src/test/rustdoc-gui", builder) { + if !p.ends_with(".goml") { + eprintln!("A non-goml file was given: `{}`", path.display()); + panic!("Cannot run rustdoc-gui tests"); + } + if let Some(name) = path.file_name().and_then(|f| f.to_str()) { + command.arg("--file").arg(name); + } + } + } + for test_arg in builder.config.cmd.test_args() { + command.arg(test_arg); + } + builder.run(&mut command); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Tidy; + +impl Step for Tidy { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + /// Runs the `tidy` tool. + /// + /// This tool in `src/tools` checks up on various bits and pieces of style and + /// otherwise just implements a few lint-like checks that are specific to the + /// compiler itself. + /// + /// Once tidy passes, this step also runs `fmt --check` if tests are being run + /// for the `dev` or `nightly` channels. + fn run(self, builder: &Builder<'_>) { + let mut cmd = builder.tool_cmd(Tool::Tidy); + cmd.arg(&builder.src); + cmd.arg(&builder.initial_cargo); + cmd.arg(&builder.out); + cmd.arg(builder.jobs().to_string()); + if builder.is_verbose() { + cmd.arg("--verbose"); + } + + builder.info("tidy check"); + try_run(builder, &mut cmd); + + if builder.config.channel == "dev" || builder.config.channel == "nightly" { + builder.info("fmt check"); + if builder.initial_rustfmt().is_none() { + let inferred_rustfmt_dir = builder.config.initial_rustc.parent().unwrap(); + eprintln!( + "\ +error: no `rustfmt` binary found in {PATH} +info: `rust.channel` is currently set to \"{CHAN}\" +help: if you are testing a beta branch, set `rust.channel` to \"beta\" in the `config.toml` file +help: to skip test's attempt to check tidiness, pass `--exclude src/tools/tidy` to `x.py test`", + PATH = inferred_rustfmt_dir.display(), + CHAN = builder.config.channel, + ); + crate::detail_exit(1); + } + crate::format::format(&builder, !builder.config.cmd.bless(), &[]); + } + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/tidy") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Tidy); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct ExpandYamlAnchors; + +impl Step for ExpandYamlAnchors { + type Output = (); + const ONLY_HOSTS: bool = true; + + /// Ensure the `generate-ci-config` tool was run locally. + /// + /// The tool in `src/tools` reads the CI definition in `src/ci/builders.yml` and generates the + /// appropriate configuration for all our CI providers. This step ensures the tool was called + /// by the user before committing CI changes. + fn run(self, builder: &Builder<'_>) { + builder.info("Ensuring the YAML anchors in the GitHub Actions config were expanded"); + try_run( + builder, + &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("check").arg(&builder.src), + ); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/expand-yaml-anchors") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(ExpandYamlAnchors); + } +} + +fn testdir(builder: &Builder<'_>, host: TargetSelection) -> PathBuf { + builder.out.join(host.triple).join("test") +} + +macro_rules! default_test { + ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr }) => { + test!($name { path: $path, mode: $mode, suite: $suite, default: true, host: false }); + }; +} + +macro_rules! default_test_with_compare_mode { + ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, + compare_mode: $compare_mode:expr }) => { + test_with_compare_mode!($name { + path: $path, + mode: $mode, + suite: $suite, + default: true, + host: false, + compare_mode: $compare_mode + }); + }; +} + +macro_rules! host_test { + ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr }) => { + test!($name { path: $path, mode: $mode, suite: $suite, default: true, host: true }); + }; +} + +macro_rules! test { + ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, default: $default:expr, + host: $host:expr }) => { + test_definitions!($name { + path: $path, + mode: $mode, + suite: $suite, + default: $default, + host: $host, + compare_mode: None + }); + }; +} + +macro_rules! test_with_compare_mode { + ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, default: $default:expr, + host: $host:expr, compare_mode: $compare_mode:expr }) => { + test_definitions!($name { + path: $path, + mode: $mode, + suite: $suite, + default: $default, + host: $host, + compare_mode: Some($compare_mode) + }); + }; +} + +macro_rules! test_definitions { + ($name:ident { + path: $path:expr, + mode: $mode:expr, + suite: $suite:expr, + default: $default:expr, + host: $host:expr, + compare_mode: $compare_mode:expr + }) => { + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] + pub struct $name { + pub compiler: Compiler, + pub target: TargetSelection, + } + + impl Step for $name { + type Output = (); + const DEFAULT: bool = $default; + const ONLY_HOSTS: bool = $host; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.suite_path($path) + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + + run.builder.ensure($name { compiler, target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + builder.ensure(Compiletest { + compiler: self.compiler, + target: self.target, + mode: $mode, + suite: $suite, + path: $path, + compare_mode: $compare_mode, + }) + } + } + }; +} + +default_test!(Ui { path: "src/test/ui", mode: "ui", suite: "ui" }); + +default_test!(RunPassValgrind { + path: "src/test/run-pass-valgrind", + mode: "run-pass-valgrind", + suite: "run-pass-valgrind" +}); + +default_test!(MirOpt { path: "src/test/mir-opt", mode: "mir-opt", suite: "mir-opt" }); + +default_test!(Codegen { path: "src/test/codegen", mode: "codegen", suite: "codegen" }); + +default_test!(CodegenUnits { + path: "src/test/codegen-units", + mode: "codegen-units", + suite: "codegen-units" +}); + +default_test!(Incremental { + path: "src/test/incremental", + mode: "incremental", + suite: "incremental" +}); + +default_test_with_compare_mode!(Debuginfo { + path: "src/test/debuginfo", + mode: "debuginfo", + suite: "debuginfo", + compare_mode: "split-dwarf" +}); + +host_test!(UiFullDeps { path: "src/test/ui-fulldeps", mode: "ui", suite: "ui-fulldeps" }); + +host_test!(Rustdoc { path: "src/test/rustdoc", mode: "rustdoc", suite: "rustdoc" }); +host_test!(RustdocUi { path: "src/test/rustdoc-ui", mode: "ui", suite: "rustdoc-ui" }); + +host_test!(RustdocJson { + path: "src/test/rustdoc-json", + mode: "rustdoc-json", + suite: "rustdoc-json" +}); + +host_test!(Pretty { path: "src/test/pretty", mode: "pretty", suite: "pretty" }); + +default_test!(RunMake { path: "src/test/run-make", mode: "run-make", suite: "run-make" }); + +host_test!(RunMakeFullDeps { + path: "src/test/run-make-fulldeps", + mode: "run-make", + suite: "run-make-fulldeps" +}); + +default_test!(Assembly { path: "src/test/assembly", mode: "assembly", suite: "assembly" }); + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +struct Compiletest { + compiler: Compiler, + target: TargetSelection, + mode: &'static str, + suite: &'static str, + path: &'static str, + compare_mode: Option<&'static str>, +} + +impl Step for Compiletest { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Executes the `compiletest` tool to run a suite of tests. + /// + /// Compiles all tests with `compiler` for `target` with the specified + /// compiletest `mode` and `suite` arguments. For example `mode` can be + /// "run-pass" or `suite` can be something like `debuginfo`. + fn run(self, builder: &Builder<'_>) { + if builder.top_stage == 0 && env::var("COMPILETEST_FORCE_STAGE0").is_err() { + eprintln!("\ +error: `--stage 0` runs compiletest on the beta compiler, not your local changes, and will almost always cause tests to fail +help: to test the compiler, use `--stage 1` instead +help: to test the standard library, use `--stage 0 library/std` instead +note: if you're sure you want to do this, please open an issue as to why. In the meantime, you can override this with `COMPILETEST_FORCE_STAGE0=1`." + ); + crate::detail_exit(1); + } + + let compiler = self.compiler; + let target = self.target; + let mode = self.mode; + let suite = self.suite; + + // Path for test suite + let suite_path = self.path; + + // Skip codegen tests if they aren't enabled in configuration. + if !builder.config.codegen_tests && suite == "codegen" { + return; + } + + if suite == "debuginfo" { + builder + .ensure(dist::DebuggerScripts { sysroot: builder.sysroot(compiler), host: target }); + } + + if suite.ends_with("fulldeps") { + builder.ensure(compile::Rustc::new(compiler, target)); + } + + builder.ensure(compile::Std::new(compiler, target)); + // ensure that `libproc_macro` is available on the host. + builder.ensure(compile::Std::new(compiler, compiler.host)); + + // Also provide `rust_test_helpers` for the host. + builder.ensure(native::TestHelpers { target: compiler.host }); + + // As well as the target, except for plain wasm32, which can't build it + if !target.contains("wasm") || target.contains("emscripten") { + builder.ensure(native::TestHelpers { target }); + } + + builder.ensure(RemoteCopyLibs { compiler, target }); + + let mut cmd = builder.tool_cmd(Tool::Compiletest); + + // compiletest currently has... a lot of arguments, so let's just pass all + // of them! + + cmd.arg("--compile-lib-path").arg(builder.rustc_libdir(compiler)); + cmd.arg("--run-lib-path").arg(builder.sysroot_libdir(compiler, target)); + cmd.arg("--rustc-path").arg(builder.rustc(compiler)); + + let is_rustdoc = suite.ends_with("rustdoc-ui") || suite.ends_with("rustdoc-js"); + + // Avoid depending on rustdoc when we don't need it. + if mode == "rustdoc" + || mode == "run-make" + || (mode == "ui" && is_rustdoc) + || mode == "js-doc-test" + || mode == "rustdoc-json" + { + cmd.arg("--rustdoc-path").arg(builder.rustdoc(compiler)); + } + + if mode == "rustdoc-json" { + // Use the beta compiler for jsondocck + let json_compiler = compiler.with_stage(0); + cmd.arg("--jsondocck-path") + .arg(builder.ensure(tool::JsonDocCk { compiler: json_compiler, target })); + } + + if mode == "run-make" { + let rust_demangler = builder + .ensure(tool::RustDemangler { + compiler, + target: compiler.host, + extra_features: Vec::new(), + }) + .expect("in-tree tool"); + cmd.arg("--rust-demangler-path").arg(rust_demangler); + } + + cmd.arg("--src-base").arg(builder.src.join("src/test").join(suite)); + cmd.arg("--build-base").arg(testdir(builder, compiler.host).join(suite)); + cmd.arg("--stage-id").arg(format!("stage{}-{}", compiler.stage, target)); + cmd.arg("--suite").arg(suite); + cmd.arg("--mode").arg(mode); + cmd.arg("--target").arg(target.rustc_target_arg()); + cmd.arg("--host").arg(&*compiler.host.triple); + cmd.arg("--llvm-filecheck").arg(builder.llvm_filecheck(builder.config.build)); + + if builder.config.cmd.bless() { + cmd.arg("--bless"); + } + + if builder.config.cmd.force_rerun() { + cmd.arg("--force-rerun"); + } + + let compare_mode = + builder.config.cmd.compare_mode().or_else(|| { + if builder.config.test_compare_mode { self.compare_mode } else { None } + }); + + if let Some(ref pass) = builder.config.cmd.pass() { + cmd.arg("--pass"); + cmd.arg(pass); + } + + if let Some(ref run) = builder.config.cmd.run() { + cmd.arg("--run"); + cmd.arg(run); + } + + if let Some(ref nodejs) = builder.config.nodejs { + cmd.arg("--nodejs").arg(nodejs); + } + if let Some(ref npm) = builder.config.npm { + cmd.arg("--npm").arg(npm); + } + if builder.config.rust_optimize_tests { + cmd.arg("--optimize-tests"); + } + let mut flags = if is_rustdoc { Vec::new() } else { vec!["-Crpath".to_string()] }; + flags.push(format!("-Cdebuginfo={}", builder.config.rust_debuginfo_level_tests)); + flags.push(builder.config.cmd.rustc_args().join(" ")); + + if let Some(linker) = builder.linker(target) { + cmd.arg("--linker").arg(linker); + } + + let mut hostflags = flags.clone(); + hostflags.push(format!("-Lnative={}", builder.test_helpers_out(compiler.host).display())); + hostflags.extend(builder.lld_flags(compiler.host)); + cmd.arg("--host-rustcflags").arg(hostflags.join(" ")); + + let mut targetflags = flags; + targetflags.push(format!("-Lnative={}", builder.test_helpers_out(target).display())); + targetflags.extend(builder.lld_flags(target)); + cmd.arg("--target-rustcflags").arg(targetflags.join(" ")); + + cmd.arg("--python").arg(builder.python()); + + if let Some(ref gdb) = builder.config.gdb { + cmd.arg("--gdb").arg(gdb); + } + + let run = |cmd: &mut Command| { + cmd.output().map(|output| { + String::from_utf8_lossy(&output.stdout) + .lines() + .next() + .unwrap_or_else(|| panic!("{:?} failed {:?}", cmd, output)) + .to_string() + }) + }; + let lldb_exe = "lldb"; + let lldb_version = Command::new(lldb_exe) + .arg("--version") + .output() + .map(|output| String::from_utf8_lossy(&output.stdout).to_string()) + .ok(); + if let Some(ref vers) = lldb_version { + cmd.arg("--lldb-version").arg(vers); + let lldb_python_dir = run(Command::new(lldb_exe).arg("-P")).ok(); + if let Some(ref dir) = lldb_python_dir { + cmd.arg("--lldb-python-dir").arg(dir); + } + } + + if util::forcing_clang_based_tests() { + let clang_exe = builder.llvm_out(target).join("bin").join("clang"); + cmd.arg("--run-clang-based-tests-with").arg(clang_exe); + } + + // Get paths from cmd args + let paths = match &builder.config.cmd { + Subcommand::Test { ref paths, .. } => &paths[..], + _ => &[], + }; + + // Get test-args by striping suite path + let mut test_args: Vec<&str> = paths + .iter() + .filter_map(|p| util::is_valid_test_suite_arg(p, suite_path, builder)) + .collect(); + + test_args.append(&mut builder.config.cmd.test_args()); + + cmd.args(&test_args); + + if builder.is_verbose() { + cmd.arg("--verbose"); + } + + if !builder.config.verbose_tests { + cmd.arg("--quiet"); + } + + let mut llvm_components_passed = false; + let mut copts_passed = false; + if builder.config.llvm_enabled() { + let llvm_config = builder.ensure(native::Llvm { target: builder.config.build }); + if !builder.config.dry_run { + let llvm_version = output(Command::new(&llvm_config).arg("--version")); + let llvm_components = output(Command::new(&llvm_config).arg("--components")); + // Remove trailing newline from llvm-config output. + cmd.arg("--llvm-version") + .arg(llvm_version.trim()) + .arg("--llvm-components") + .arg(llvm_components.trim()); + llvm_components_passed = true; + } + if !builder.is_rust_llvm(target) { + cmd.arg("--system-llvm"); + } + + // Tests that use compiler libraries may inherit the `-lLLVM` link + // requirement, but the `-L` library path is not propagated across + // separate compilations. We can add LLVM's library path to the + // platform-specific environment variable as a workaround. + if !builder.config.dry_run && suite.ends_with("fulldeps") { + let llvm_libdir = output(Command::new(&llvm_config).arg("--libdir")); + add_link_lib_path(vec![llvm_libdir.trim().into()], &mut cmd); + } + + // Only pass correct values for these flags for the `run-make` suite as it + // requires that a C++ compiler was configured which isn't always the case. + if !builder.config.dry_run && matches!(suite, "run-make" | "run-make-fulldeps") { + // The llvm/bin directory contains many useful cross-platform + // tools. Pass the path to run-make tests so they can use them. + let llvm_bin_path = llvm_config + .parent() + .expect("Expected llvm-config to be contained in directory"); + assert!(llvm_bin_path.is_dir()); + cmd.arg("--llvm-bin-dir").arg(llvm_bin_path); + + // If LLD is available, add it to the PATH + if builder.config.lld_enabled { + let lld_install_root = + builder.ensure(native::Lld { target: builder.config.build }); + + let lld_bin_path = lld_install_root.join("bin"); + + let old_path = env::var_os("PATH").unwrap_or_default(); + let new_path = env::join_paths( + std::iter::once(lld_bin_path).chain(env::split_paths(&old_path)), + ) + .expect("Could not add LLD bin path to PATH"); + cmd.env("PATH", new_path); + } + } + } + + // Only pass correct values for these flags for the `run-make` suite as it + // requires that a C++ compiler was configured which isn't always the case. + if !builder.config.dry_run && matches!(suite, "run-make" | "run-make-fulldeps") { + cmd.arg("--cc") + .arg(builder.cc(target)) + .arg("--cxx") + .arg(builder.cxx(target).unwrap()) + .arg("--cflags") + .arg(builder.cflags(target, GitRepo::Rustc, CLang::C).join(" ")) + .arg("--cxxflags") + .arg(builder.cflags(target, GitRepo::Rustc, CLang::Cxx).join(" ")); + copts_passed = true; + if let Some(ar) = builder.ar(target) { + cmd.arg("--ar").arg(ar); + } + } + + if !llvm_components_passed { + cmd.arg("--llvm-components").arg(""); + } + if !copts_passed { + cmd.arg("--cc") + .arg("") + .arg("--cxx") + .arg("") + .arg("--cflags") + .arg("") + .arg("--cxxflags") + .arg(""); + } + + if builder.remote_tested(target) { + cmd.arg("--remote-test-client").arg(builder.tool_exe(Tool::RemoteTestClient)); + } + + // Running a C compiler on MSVC requires a few env vars to be set, to be + // sure to set them here. + // + // Note that if we encounter `PATH` we make sure to append to our own `PATH` + // rather than stomp over it. + if target.contains("msvc") { + for &(ref k, ref v) in builder.cc[&target].env() { + if k != "PATH" { + cmd.env(k, v); + } + } + } + cmd.env("RUSTC_BOOTSTRAP", "1"); + // Override the rustc version used in symbol hashes to reduce the amount of normalization + // needed when diffing test output. + cmd.env("RUSTC_FORCE_RUSTC_VERSION", "compiletest"); + cmd.env("DOC_RUST_LANG_ORG_CHANNEL", builder.doc_rust_lang_org_channel()); + builder.add_rust_test_threads(&mut cmd); + + if builder.config.sanitizers_enabled(target) { + cmd.env("RUSTC_SANITIZER_SUPPORT", "1"); + } + + if builder.config.profiler_enabled(target) { + cmd.env("RUSTC_PROFILER_SUPPORT", "1"); + } + + cmd.env("RUST_TEST_TMPDIR", builder.tempdir()); + + cmd.arg("--adb-path").arg("adb"); + cmd.arg("--adb-test-dir").arg(ADB_TEST_DIR); + if target.contains("android") { + // Assume that cc for this target comes from the android sysroot + cmd.arg("--android-cross-path") + .arg(builder.cc(target).parent().unwrap().parent().unwrap()); + } else { + cmd.arg("--android-cross-path").arg(""); + } + + if builder.config.cmd.rustfix_coverage() { + cmd.arg("--rustfix-coverage"); + } + + cmd.env("BOOTSTRAP_CARGO", &builder.initial_cargo); + + cmd.arg("--channel").arg(&builder.config.channel); + + builder.ci_env.force_coloring_in_ci(&mut cmd); + + builder.info(&format!( + "Check compiletest suite={} mode={} ({} -> {})", + suite, mode, &compiler.host, target + )); + let _time = util::timeit(&builder); + try_run(builder, &mut cmd); + + if let Some(compare_mode) = compare_mode { + cmd.arg("--compare-mode").arg(compare_mode); + builder.info(&format!( + "Check compiletest suite={} mode={} compare_mode={} ({} -> {})", + suite, mode, compare_mode, &compiler.host, target + )); + let _time = util::timeit(&builder); + try_run(builder, &mut cmd); + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct BookTest { + compiler: Compiler, + path: PathBuf, + name: &'static str, + is_ext_doc: bool, +} + +impl Step for BookTest { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Runs the documentation tests for a book in `src/doc`. + /// + /// This uses the `rustdoc` that sits next to `compiler`. + fn run(self, builder: &Builder<'_>) { + // External docs are different from local because: + // - Some books need pre-processing by mdbook before being tested. + // - They need to save their state to toolstate. + // - They are only tested on the "checktools" builders. + // + // The local docs are tested by default, and we don't want to pay the + // cost of building mdbook, so they use `rustdoc --test` directly. + // Also, the unstable book is special because SUMMARY.md is generated, + // so it is easier to just run `rustdoc` on its files. + if self.is_ext_doc { + self.run_ext_doc(builder); + } else { + self.run_local_doc(builder); + } + } +} + +impl BookTest { + /// This runs the equivalent of `mdbook test` (via the rustbook wrapper) + /// which in turn runs `rustdoc --test` on each file in the book. + fn run_ext_doc(self, builder: &Builder<'_>) { + let compiler = self.compiler; + + builder.ensure(compile::Std::new(compiler, compiler.host)); + + // mdbook just executes a binary named "rustdoc", so we need to update + // PATH so that it points to our rustdoc. + let mut rustdoc_path = builder.rustdoc(compiler); + rustdoc_path.pop(); + let old_path = env::var_os("PATH").unwrap_or_default(); + let new_path = env::join_paths(iter::once(rustdoc_path).chain(env::split_paths(&old_path))) + .expect("could not add rustdoc to PATH"); + + let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook); + let path = builder.src.join(&self.path); + // Books often have feature-gated example text. + rustbook_cmd.env("RUSTC_BOOTSTRAP", "1"); + rustbook_cmd.env("PATH", new_path).arg("test").arg(path); + builder.add_rust_test_threads(&mut rustbook_cmd); + builder.info(&format!("Testing rustbook {}", self.path.display())); + let _time = util::timeit(&builder); + let toolstate = if try_run(builder, &mut rustbook_cmd) { + ToolState::TestPass + } else { + ToolState::TestFail + }; + builder.save_toolstate(self.name, toolstate); + } + + /// This runs `rustdoc --test` on all `.md` files in the path. + fn run_local_doc(self, builder: &Builder<'_>) { + let compiler = self.compiler; + + builder.ensure(compile::Std::new(compiler, compiler.host)); + + // Do a breadth-first traversal of the `src/doc` directory and just run + // tests for all files that end in `*.md` + let mut stack = vec![builder.src.join(self.path)]; + let _time = util::timeit(&builder); + let mut files = Vec::new(); + while let Some(p) = stack.pop() { + if p.is_dir() { + stack.extend(t!(p.read_dir()).map(|p| t!(p).path())); + continue; + } + + if p.extension().and_then(|s| s.to_str()) != Some("md") { + continue; + } + + files.push(p); + } + + files.sort(); + + for file in files { + markdown_test(builder, compiler, &file); + } + } +} + +macro_rules! test_book { + ($($name:ident, $path:expr, $book_name:expr, default=$default:expr;)+) => { + $( + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] + pub struct $name { + compiler: Compiler, + } + + impl Step for $name { + type Output = (); + const DEFAULT: bool = $default; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path($path) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure($name { + compiler: run.builder.compiler(run.builder.top_stage, run.target), + }); + } + + fn run(self, builder: &Builder<'_>) { + builder.ensure(BookTest { + compiler: self.compiler, + path: PathBuf::from($path), + name: $book_name, + is_ext_doc: !$default, + }); + } + } + )+ + } +} + +test_book!( + Nomicon, "src/doc/nomicon", "nomicon", default=false; + Reference, "src/doc/reference", "reference", default=false; + RustdocBook, "src/doc/rustdoc", "rustdoc", default=true; + RustcBook, "src/doc/rustc", "rustc", default=true; + RustByExample, "src/doc/rust-by-example", "rust-by-example", default=false; + EmbeddedBook, "src/doc/embedded-book", "embedded-book", default=false; + TheBook, "src/doc/book", "book", default=false; + UnstableBook, "src/doc/unstable-book", "unstable-book", default=true; + EditionGuide, "src/doc/edition-guide", "edition-guide", default=false; +); + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct ErrorIndex { + compiler: Compiler, +} + +impl Step for ErrorIndex { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/error_index_generator") + } + + fn make_run(run: RunConfig<'_>) { + // error_index_generator depends on librustdoc. Use the compiler that + // is normally used to build rustdoc for other tests (like compiletest + // tests in src/test/rustdoc) so that it shares the same artifacts. + let compiler = run.builder.compiler(run.builder.top_stage, run.builder.config.build); + run.builder.ensure(ErrorIndex { compiler }); + } + + /// Runs the error index generator tool to execute the tests located in the error + /// index. + /// + /// The `error_index_generator` tool lives in `src/tools` and is used to + /// generate a markdown file from the error indexes of the code base which is + /// then passed to `rustdoc --test`. + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + + let dir = testdir(builder, compiler.host); + t!(fs::create_dir_all(&dir)); + let output = dir.join("error-index.md"); + + let mut tool = tool::ErrorIndex::command(builder); + tool.arg("markdown").arg(&output); + + builder.info(&format!("Testing error-index stage{}", compiler.stage)); + let _time = util::timeit(&builder); + builder.run_quiet(&mut tool); + // The tests themselves need to link to std, so make sure it is + // available. + builder.ensure(compile::Std::new(compiler, compiler.host)); + markdown_test(builder, compiler, &output); + } +} + +fn markdown_test(builder: &Builder<'_>, compiler: Compiler, markdown: &Path) -> bool { + if let Ok(contents) = fs::read_to_string(markdown) { + if !contents.contains("```") { + return true; + } + } + + builder.info(&format!("doc tests for: {}", markdown.display())); + let mut cmd = builder.rustdoc_cmd(compiler); + builder.add_rust_test_threads(&mut cmd); + // allow for unstable options such as new editions + cmd.arg("-Z"); + cmd.arg("unstable-options"); + cmd.arg("--test"); + cmd.arg(markdown); + cmd.env("RUSTC_BOOTSTRAP", "1"); + + let test_args = builder.config.cmd.test_args().join(" "); + cmd.arg("--test-args").arg(test_args); + + if builder.config.verbose_tests { + try_run(builder, &mut cmd) + } else { + try_run_quiet(builder, &mut cmd) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct RustcGuide; + +impl Step for RustcGuide { + type Output = (); + const DEFAULT: bool = false; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/doc/rustc-dev-guide") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustcGuide); + } + + fn run(self, builder: &Builder<'_>) { + let relative_path = Path::new("src").join("doc").join("rustc-dev-guide"); + builder.update_submodule(&relative_path); + + let src = builder.src.join(relative_path); + let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook); + let toolstate = if try_run(builder, rustbook_cmd.arg("linkcheck").arg(&src)) { + ToolState::TestPass + } else { + ToolState::TestFail + }; + builder.save_toolstate("rustc-dev-guide", toolstate); + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CrateLibrustc { + compiler: Compiler, + target: TargetSelection, + test_kind: TestKind, + crates: Vec<Interned<String>>, +} + +impl Step for CrateLibrustc { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.crate_or_deps("rustc-main") + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + let host = run.build_triple(); + let compiler = builder.compiler_for(builder.top_stage, host, host); + let crates = run + .paths + .iter() + .map(|p| builder.crate_paths[&p.assert_single_path().path].clone()) + .collect(); + let test_kind = builder.kind.into(); + + builder.ensure(CrateLibrustc { compiler, target: run.target, test_kind, crates }); + } + + fn run(self, builder: &Builder<'_>) { + builder.ensure(Crate { + compiler: self.compiler, + target: self.target, + mode: Mode::Rustc, + test_kind: self.test_kind, + crates: self.crates, + }); + } +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Crate { + pub compiler: Compiler, + pub target: TargetSelection, + pub mode: Mode, + pub test_kind: TestKind, + pub crates: Vec<Interned<String>>, +} + +impl Step for Crate { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.crate_or_deps("test") + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + let host = run.build_triple(); + let compiler = builder.compiler_for(builder.top_stage, host, host); + let test_kind = builder.kind.into(); + let crates = run + .paths + .iter() + .map(|p| builder.crate_paths[&p.assert_single_path().path].clone()) + .collect(); + + builder.ensure(Crate { compiler, target: run.target, mode: Mode::Std, test_kind, crates }); + } + + /// Runs all unit tests plus documentation tests for a given crate defined + /// by a `Cargo.toml` (single manifest) + /// + /// This is what runs tests for crates like the standard library, compiler, etc. + /// It essentially is the driver for running `cargo test`. + /// + /// Currently this runs all tests for a DAG by passing a bunch of `-p foo` + /// arguments, and those arguments are discovered from `cargo metadata`. + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target = self.target; + let mode = self.mode; + let test_kind = self.test_kind; + + builder.ensure(compile::Std::new(compiler, target)); + builder.ensure(RemoteCopyLibs { compiler, target }); + + // If we're not doing a full bootstrap but we're testing a stage2 + // version of libstd, then what we're actually testing is the libstd + // produced in stage1. Reflect that here by updating the compiler that + // we're working with automatically. + let compiler = builder.compiler_for(compiler.stage, compiler.host, target); + + let mut cargo = + builder.cargo(compiler, mode, SourceType::InTree, target, test_kind.subcommand()); + match mode { + Mode::Std => { + compile::std_cargo(builder, target, compiler.stage, &mut cargo); + } + Mode::Rustc => { + compile::rustc_cargo(builder, &mut cargo, target); + } + _ => panic!("can only test libraries"), + }; + + // Build up the base `cargo test` command. + // + // Pass in some standard flags then iterate over the graph we've discovered + // in `cargo metadata` with the maps above and figure out what `-p` + // arguments need to get passed. + if test_kind.subcommand() == "test" && !builder.fail_fast { + cargo.arg("--no-fail-fast"); + } + match builder.doc_tests { + DocTests::Only => { + cargo.arg("--doc"); + } + DocTests::No => { + cargo.args(&["--lib", "--bins", "--examples", "--tests", "--benches"]); + } + DocTests::Yes => {} + } + + for krate in &self.crates { + cargo.arg("-p").arg(krate); + } + + // The tests are going to run with the *target* libraries, so we need to + // ensure that those libraries show up in the LD_LIBRARY_PATH equivalent. + // + // Note that to run the compiler we need to run with the *host* libraries, + // but our wrapper scripts arrange for that to be the case anyway. + let mut dylib_path = dylib_path(); + dylib_path.insert(0, PathBuf::from(&*builder.sysroot_libdir(compiler, target))); + cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); + + cargo.arg("--"); + cargo.args(&builder.config.cmd.test_args()); + + if !builder.config.verbose_tests { + cargo.arg("--quiet"); + } + + if target.contains("emscripten") { + cargo.env( + format!("CARGO_TARGET_{}_RUNNER", envify(&target.triple)), + builder.config.nodejs.as_ref().expect("nodejs not configured"), + ); + } else if target.starts_with("wasm32") { + let node = builder.config.nodejs.as_ref().expect("nodejs not configured"); + let runner = + format!("{} {}/src/etc/wasm32-shim.js", node.display(), builder.src.display()); + cargo.env(format!("CARGO_TARGET_{}_RUNNER", envify(&target.triple)), &runner); + } else if builder.remote_tested(target) { + cargo.env( + format!("CARGO_TARGET_{}_RUNNER", envify(&target.triple)), + format!("{} run 0", builder.tool_exe(Tool::RemoteTestClient).display()), + ); + } + + builder.info(&format!( + "{} {:?} stage{} ({} -> {})", + test_kind, self.crates, compiler.stage, &compiler.host, target + )); + let _time = util::timeit(&builder); + try_run(builder, &mut cargo.into()); + } +} + +/// Rustdoc is special in various ways, which is why this step is different from `Crate`. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct CrateRustdoc { + host: TargetSelection, + test_kind: TestKind, +} + +impl Step for CrateRustdoc { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.paths(&["src/librustdoc", "src/tools/rustdoc"]) + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + + let test_kind = builder.kind.into(); + + builder.ensure(CrateRustdoc { host: run.target, test_kind }); + } + + fn run(self, builder: &Builder<'_>) { + let test_kind = self.test_kind; + let target = self.host; + + let compiler = if builder.download_rustc() { + builder.compiler(builder.top_stage, target) + } else { + // Use the previous stage compiler to reuse the artifacts that are + // created when running compiletest for src/test/rustdoc. If this used + // `compiler`, then it would cause rustdoc to be built *again*, which + // isn't really necessary. + builder.compiler_for(builder.top_stage, target, target) + }; + builder.ensure(compile::Rustc::new(compiler, target)); + + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + target, + test_kind.subcommand(), + "src/tools/rustdoc", + SourceType::InTree, + &[], + ); + if test_kind.subcommand() == "test" && !builder.fail_fast { + cargo.arg("--no-fail-fast"); + } + match builder.doc_tests { + DocTests::Only => { + cargo.arg("--doc"); + } + DocTests::No => { + cargo.args(&["--lib", "--bins", "--examples", "--tests", "--benches"]); + } + DocTests::Yes => {} + } + + cargo.arg("-p").arg("rustdoc:0.0.0"); + + cargo.arg("--"); + cargo.args(&builder.config.cmd.test_args()); + + if self.host.contains("musl") { + cargo.arg("'-Ctarget-feature=-crt-static'"); + } + + // This is needed for running doctests on librustdoc. This is a bit of + // an unfortunate interaction with how bootstrap works and how cargo + // sets up the dylib path, and the fact that the doctest (in + // html/markdown.rs) links to rustc-private libs. For stage1, the + // compiler host dylibs (in stage1/lib) are not the same as the target + // dylibs (in stage1/lib/rustlib/...). This is different from a normal + // rust distribution where they are the same. + // + // On the cargo side, normal tests use `target_process` which handles + // setting up the dylib for a *target* (stage1/lib/rustlib/... in this + // case). However, for doctests it uses `rustdoc_process` which only + // sets up the dylib path for the *host* (stage1/lib), which is the + // wrong directory. + // + // Recall that we special-cased `compiler_for(top_stage)` above, so we always use stage1. + // + // It should be considered to just stop running doctests on + // librustdoc. There is only one test, and it doesn't look too + // important. There might be other ways to avoid this, but it seems + // pretty convoluted. + // + // See also https://github.com/rust-lang/rust/issues/13983 where the + // host vs target dylibs for rustdoc are consistently tricky to deal + // with. + // + // Note that this set the host libdir for `download_rustc`, which uses a normal rust distribution. + let libdir = if builder.download_rustc() { + builder.rustc_libdir(compiler) + } else { + builder.sysroot_libdir(compiler, target).to_path_buf() + }; + let mut dylib_path = dylib_path(); + dylib_path.insert(0, PathBuf::from(&*libdir)); + cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); + + if !builder.config.verbose_tests { + cargo.arg("--quiet"); + } + + builder.info(&format!( + "{} rustdoc stage{} ({} -> {})", + test_kind, compiler.stage, &compiler.host, target + )); + let _time = util::timeit(&builder); + + try_run(builder, &mut cargo.into()); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct CrateRustdocJsonTypes { + host: TargetSelection, + test_kind: TestKind, +} + +impl Step for CrateRustdocJsonTypes { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/rustdoc-json-types") + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + + let test_kind = builder.kind.into(); + + builder.ensure(CrateRustdocJsonTypes { host: run.target, test_kind }); + } + + fn run(self, builder: &Builder<'_>) { + let test_kind = self.test_kind; + let target = self.host; + + // Use the previous stage compiler to reuse the artifacts that are + // created when running compiletest for src/test/rustdoc. If this used + // `compiler`, then it would cause rustdoc to be built *again*, which + // isn't really necessary. + let compiler = builder.compiler_for(builder.top_stage, target, target); + builder.ensure(compile::Rustc::new(compiler, target)); + + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + target, + test_kind.subcommand(), + "src/rustdoc-json-types", + SourceType::InTree, + &[], + ); + if test_kind.subcommand() == "test" && !builder.fail_fast { + cargo.arg("--no-fail-fast"); + } + + cargo.arg("-p").arg("rustdoc-json-types"); + + cargo.arg("--"); + cargo.args(&builder.config.cmd.test_args()); + + if self.host.contains("musl") { + cargo.arg("'-Ctarget-feature=-crt-static'"); + } + + if !builder.config.verbose_tests { + cargo.arg("--quiet"); + } + + builder.info(&format!( + "{} rustdoc-json-types stage{} ({} -> {})", + test_kind, compiler.stage, &compiler.host, target + )); + let _time = util::timeit(&builder); + + try_run(builder, &mut cargo.into()); + } +} + +/// Some test suites are run inside emulators or on remote devices, and most +/// of our test binaries are linked dynamically which means we need to ship +/// the standard library and such to the emulator ahead of time. This step +/// represents this and is a dependency of all test suites. +/// +/// Most of the time this is a no-op. For some steps such as shipping data to +/// QEMU we have to build our own tools so we've got conditional dependencies +/// on those programs as well. Note that the remote test client is built for +/// the build target (us) and the server is built for the target. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct RemoteCopyLibs { + compiler: Compiler, + target: TargetSelection, +} + +impl Step for RemoteCopyLibs { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target = self.target; + if !builder.remote_tested(target) { + return; + } + + builder.ensure(compile::Std::new(compiler, target)); + + builder.info(&format!("REMOTE copy libs to emulator ({})", target)); + + let server = builder.ensure(tool::RemoteTestServer { compiler, target }); + + // Spawn the emulator and wait for it to come online + let tool = builder.tool_exe(Tool::RemoteTestClient); + let mut cmd = Command::new(&tool); + cmd.arg("spawn-emulator").arg(target.triple).arg(&server).arg(builder.tempdir()); + if let Some(rootfs) = builder.qemu_rootfs(target) { + cmd.arg(rootfs); + } + builder.run(&mut cmd); + + // Push all our dylibs to the emulator + for f in t!(builder.sysroot_libdir(compiler, target).read_dir()) { + let f = t!(f); + let name = f.file_name().into_string().unwrap(); + if util::is_dylib(&name) { + builder.run(Command::new(&tool).arg("push").arg(f.path())); + } + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Distcheck; + +impl Step for Distcheck { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("distcheck") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Distcheck); + } + + /// Runs "distcheck", a 'make check' from a tarball + fn run(self, builder: &Builder<'_>) { + builder.info("Distcheck"); + let dir = builder.tempdir().join("distcheck"); + let _ = fs::remove_dir_all(&dir); + t!(fs::create_dir_all(&dir)); + + // Guarantee that these are built before we begin running. + builder.ensure(dist::PlainSourceTarball); + builder.ensure(dist::Src); + + let mut cmd = Command::new("tar"); + cmd.arg("-xf") + .arg(builder.ensure(dist::PlainSourceTarball).tarball()) + .arg("--strip-components=1") + .current_dir(&dir); + builder.run(&mut cmd); + builder.run( + Command::new("./configure") + .args(&builder.config.configure_args) + .arg("--enable-vendor") + .current_dir(&dir), + ); + builder.run( + Command::new(util::make(&builder.config.build.triple)).arg("check").current_dir(&dir), + ); + + // Now make sure that rust-src has all of libstd's dependencies + builder.info("Distcheck rust-src"); + let dir = builder.tempdir().join("distcheck-src"); + let _ = fs::remove_dir_all(&dir); + t!(fs::create_dir_all(&dir)); + + let mut cmd = Command::new("tar"); + cmd.arg("-xf") + .arg(builder.ensure(dist::Src).tarball()) + .arg("--strip-components=1") + .current_dir(&dir); + builder.run(&mut cmd); + + let toml = dir.join("rust-src/lib/rustlib/src/rust/library/std/Cargo.toml"); + builder.run( + Command::new(&builder.initial_cargo) + .arg("generate-lockfile") + .arg("--manifest-path") + .arg(&toml) + .current_dir(&dir), + ); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Bootstrap; + +impl Step for Bootstrap { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + /// Tests the build system itself. + fn run(self, builder: &Builder<'_>) { + let mut check_bootstrap = Command::new(&builder.python()); + check_bootstrap.arg("bootstrap_test.py").current_dir(builder.src.join("src/bootstrap/")); + try_run(builder, &mut check_bootstrap); + + let mut cmd = Command::new(&builder.initial_cargo); + cmd.arg("test") + .current_dir(builder.src.join("src/bootstrap")) + .env("RUSTFLAGS", "-Cdebuginfo=2") + .env("CARGO_TARGET_DIR", builder.out.join("bootstrap")) + .env("RUSTC_BOOTSTRAP", "1") + .env("RUSTDOC", builder.rustdoc(builder.compiler(0, builder.build.build))) + .env("RUSTC", &builder.initial_rustc); + if let Some(flags) = option_env!("RUSTFLAGS") { + // Use the same rustc flags for testing as for "normal" compilation, + // so that Cargo doesn’t recompile the entire dependency graph every time: + // https://github.com/rust-lang/rust/issues/49215 + cmd.env("RUSTFLAGS", flags); + } + if !builder.fail_fast { + cmd.arg("--no-fail-fast"); + } + match builder.doc_tests { + DocTests::Only => { + cmd.arg("--doc"); + } + DocTests::No => { + cmd.args(&["--lib", "--bins", "--examples", "--tests", "--benches"]); + } + DocTests::Yes => {} + } + + cmd.arg("--").args(&builder.config.cmd.test_args()); + // rustbuild tests are racy on directory creation so just run them one at a time. + // Since there's not many this shouldn't be a problem. + cmd.arg("--test-threads=1"); + try_run(builder, &mut cmd); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/bootstrap") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Bootstrap); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct TierCheck { + pub compiler: Compiler, +} + +impl Step for TierCheck { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/tier-check") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = + run.builder.compiler_for(run.builder.top_stage, run.builder.build.build, run.target); + run.builder.ensure(TierCheck { compiler }); + } + + /// Tests the Platform Support page in the rustc book. + fn run(self, builder: &Builder<'_>) { + builder.ensure(compile::Std::new(self.compiler, self.compiler.host)); + let mut cargo = tool::prepare_tool_cargo( + builder, + self.compiler, + Mode::ToolStd, + self.compiler.host, + "run", + "src/tools/tier-check", + SourceType::InTree, + &[], + ); + cargo.arg(builder.src.join("src/doc/rustc/src/platform-support.md")); + cargo.arg(&builder.rustc(self.compiler)); + if builder.is_verbose() { + cargo.arg("--verbose"); + } + + builder.info("platform support check"); + try_run(builder, &mut cargo.into()); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct LintDocs { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for LintDocs { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/lint-docs") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(LintDocs { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + /// Tests that the lint examples in the rustc book generate the correct + /// lints and have the expected format. + fn run(self, builder: &Builder<'_>) { + builder.ensure(crate::doc::RustcBook { + compiler: self.compiler, + target: self.target, + validate: true, + }); + } +} diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs new file mode 100644 index 000000000..06fa5039f --- /dev/null +++ b/src/bootstrap/tool.rs @@ -0,0 +1,918 @@ +use std::collections::HashSet; +use std::env; +use std::fs; +use std::path::PathBuf; +use std::process::Command; + +use crate::builder::{Builder, Cargo as CargoCommand, RunConfig, ShouldRun, Step}; +use crate::channel::GitInfo; +use crate::compile; +use crate::config::TargetSelection; +use crate::toolstate::ToolState; +use crate::util::{add_dylib_path, exe, t}; +use crate::Compiler; +use crate::Mode; + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub enum SourceType { + InTree, + Submodule, +} + +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +struct ToolBuild { + compiler: Compiler, + target: TargetSelection, + tool: &'static str, + path: &'static str, + mode: Mode, + is_optional_tool: bool, + source_type: SourceType, + extra_features: Vec<String>, +} + +impl Step for ToolBuild { + type Output = Option<PathBuf>; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Builds a tool in `src/tools` + /// + /// This will build the specified tool with the specified `host` compiler in + /// `stage` into the normal cargo output directory. + fn run(self, builder: &Builder<'_>) -> Option<PathBuf> { + let compiler = self.compiler; + let target = self.target; + let mut tool = self.tool; + let path = self.path; + let is_optional_tool = self.is_optional_tool; + + match self.mode { + Mode::ToolRustc => { + builder.ensure(compile::Std::new(compiler, compiler.host)); + builder.ensure(compile::Rustc::new(compiler, target)); + } + Mode::ToolStd => builder.ensure(compile::Std::new(compiler, target)), + Mode::ToolBootstrap => {} // uses downloaded stage0 compiler libs + _ => panic!("unexpected Mode for tool build"), + } + + let cargo = prepare_tool_cargo( + builder, + compiler, + self.mode, + target, + "build", + path, + self.source_type, + &self.extra_features, + ); + + builder.info(&format!("Building stage{} tool {} ({})", compiler.stage, tool, target)); + let mut duplicates = Vec::new(); + let is_expected = compile::stream_cargo(builder, cargo, vec![], &mut |msg| { + // Only care about big things like the RLS/Cargo for now + match tool { + "rls" | "cargo" | "clippy-driver" | "miri" | "rustfmt" => {} + + _ => return, + } + let (id, features, filenames) = match msg { + compile::CargoMessage::CompilerArtifact { + package_id, + features, + filenames, + target: _, + } => (package_id, features, filenames), + _ => return, + }; + let features = features.iter().map(|s| s.to_string()).collect::<Vec<_>>(); + + for path in filenames { + let val = (tool, PathBuf::from(&*path), features.clone()); + // we're only interested in deduplicating rlibs for now + if val.1.extension().and_then(|s| s.to_str()) != Some("rlib") { + continue; + } + + // Don't worry about compiles that turn out to be host + // dependencies or build scripts. To skip these we look for + // anything that goes in `.../release/deps` but *doesn't* go in + // `$target/release/deps`. This ensure that outputs in + // `$target/release` are still considered candidates for + // deduplication. + if let Some(parent) = val.1.parent() { + if parent.ends_with("release/deps") { + let maybe_target = parent + .parent() + .and_then(|p| p.parent()) + .and_then(|p| p.file_name()) + .and_then(|p| p.to_str()) + .unwrap(); + if maybe_target != &*target.triple { + continue; + } + } + } + + // Record that we've built an artifact for `id`, and if one was + // already listed then we need to see if we reused the same + // artifact or produced a duplicate. + let mut artifacts = builder.tool_artifacts.borrow_mut(); + let prev_artifacts = artifacts.entry(target).or_default(); + let prev = match prev_artifacts.get(&*id) { + Some(prev) => prev, + None => { + prev_artifacts.insert(id.to_string(), val); + continue; + } + }; + if prev.1 == val.1 { + return; // same path, same artifact + } + + // If the paths are different and one of them *isn't* inside of + // `release/deps`, then it means it's probably in + // `$target/release`, or it's some final artifact like + // `libcargo.rlib`. In these situations Cargo probably just + // copied it up from `$target/release/deps/libcargo-xxxx.rlib`, + // so if the features are equal we can just skip it. + let prev_no_hash = prev.1.parent().unwrap().ends_with("release/deps"); + let val_no_hash = val.1.parent().unwrap().ends_with("release/deps"); + if prev.2 == val.2 || !prev_no_hash || !val_no_hash { + return; + } + + // ... and otherwise this looks like we duplicated some sort of + // compilation, so record it to generate an error later. + duplicates.push((id.to_string(), val, prev.clone())); + } + }); + + if is_expected && !duplicates.is_empty() { + eprintln!( + "duplicate artifacts found when compiling a tool, this \ + typically means that something was recompiled because \ + a transitive dependency has different features activated \ + than in a previous build:\n" + ); + let (same, different): (Vec<_>, Vec<_>) = + duplicates.into_iter().partition(|(_, cur, prev)| cur.2 == prev.2); + if !same.is_empty() { + eprintln!( + "the following dependencies are duplicated although they \ + have the same features enabled:" + ); + for (id, cur, prev) in same { + eprintln!(" {}", id); + // same features + eprintln!(" `{}` ({:?})\n `{}` ({:?})", cur.0, cur.1, prev.0, prev.1); + } + } + if !different.is_empty() { + eprintln!("the following dependencies have different features:"); + for (id, cur, prev) in different { + eprintln!(" {}", id); + let cur_features: HashSet<_> = cur.2.into_iter().collect(); + let prev_features: HashSet<_> = prev.2.into_iter().collect(); + eprintln!( + " `{}` additionally enabled features {:?} at {:?}", + cur.0, + &cur_features - &prev_features, + cur.1 + ); + eprintln!( + " `{}` additionally enabled features {:?} at {:?}", + prev.0, + &prev_features - &cur_features, + prev.1 + ); + } + } + eprintln!(); + eprintln!( + "to fix this you will probably want to edit the local \ + src/tools/rustc-workspace-hack/Cargo.toml crate, as \ + that will update the dependency graph to ensure that \ + these crates all share the same feature set" + ); + panic!("tools should not compile multiple copies of the same crate"); + } + + builder.save_toolstate( + tool, + if is_expected { ToolState::TestFail } else { ToolState::BuildFail }, + ); + + if !is_expected { + if !is_optional_tool { + crate::detail_exit(1); + } else { + None + } + } else { + // HACK(#82501): on Windows, the tools directory gets added to PATH when running tests, and + // compiletest confuses HTML tidy with the in-tree tidy. Name the in-tree tidy something + // different so the problem doesn't come up. + if tool == "tidy" { + tool = "rust-tidy"; + } + let cargo_out = builder.cargo_out(compiler, self.mode, target).join(exe(tool, target)); + let bin = builder.tools_dir(compiler).join(exe(tool, target)); + builder.copy(&cargo_out, &bin); + Some(bin) + } + } +} + +pub fn prepare_tool_cargo( + builder: &Builder<'_>, + compiler: Compiler, + mode: Mode, + target: TargetSelection, + command: &'static str, + path: &'static str, + source_type: SourceType, + extra_features: &[String], +) -> CargoCommand { + let mut cargo = builder.cargo(compiler, mode, source_type, target, command); + let dir = builder.src.join(path); + cargo.arg("--manifest-path").arg(dir.join("Cargo.toml")); + + let mut features = extra_features.to_vec(); + if builder.build.config.cargo_native_static { + if path.ends_with("cargo") + || path.ends_with("rls") + || path.ends_with("clippy") + || path.ends_with("miri") + || path.ends_with("rustfmt") + { + cargo.env("LIBZ_SYS_STATIC", "1"); + features.push("rustc-workspace-hack/all-static".to_string()); + } + } + + // clippy tests need to know about the stage sysroot. Set them consistently while building to + // avoid rebuilding when running tests. + cargo.env("SYSROOT", builder.sysroot(compiler)); + + // if tools are using lzma we want to force the build script to build its + // own copy + cargo.env("LZMA_API_STATIC", "1"); + + // CFG_RELEASE is needed by rustfmt (and possibly other tools) which + // import rustc-ap-rustc_attr which requires this to be set for the + // `#[cfg(version(...))]` attribute. + cargo.env("CFG_RELEASE", builder.rust_release()); + cargo.env("CFG_RELEASE_CHANNEL", &builder.config.channel); + cargo.env("CFG_VERSION", builder.rust_version()); + cargo.env("CFG_RELEASE_NUM", &builder.version); + cargo.env("DOC_RUST_LANG_ORG_CHANNEL", builder.doc_rust_lang_org_channel()); + + let info = GitInfo::new(builder.config.ignore_git, &dir); + if let Some(sha) = info.sha() { + cargo.env("CFG_COMMIT_HASH", sha); + } + if let Some(sha_short) = info.sha_short() { + cargo.env("CFG_SHORT_COMMIT_HASH", sha_short); + } + if let Some(date) = info.commit_date() { + cargo.env("CFG_COMMIT_DATE", date); + } + if !features.is_empty() { + cargo.arg("--features").arg(&features.join(", ")); + } + cargo +} + +macro_rules! bootstrap_tool { + ($( + $name:ident, $path:expr, $tool_name:expr + $(,is_external_tool = $external:expr)* + $(,is_unstable_tool = $unstable:expr)* + ; + )+) => { + #[derive(Copy, PartialEq, Eq, Clone)] + pub enum Tool { + $( + $name, + )+ + } + + impl<'a> Builder<'a> { + pub fn tool_exe(&self, tool: Tool) -> PathBuf { + match tool { + $(Tool::$name => + self.ensure($name { + compiler: self.compiler(0, self.config.build), + target: self.config.build, + }), + )+ + } + } + } + + $( + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] + pub struct $name { + pub compiler: Compiler, + pub target: TargetSelection, + } + + impl Step for $name { + type Output = PathBuf; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path($path) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure($name { + // snapshot compiler + compiler: run.builder.compiler(0, run.builder.config.build), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: $tool_name, + mode: if false $(|| $unstable)* { + // use in-tree libraries for unstable features + Mode::ToolStd + } else { + Mode::ToolBootstrap + }, + path: $path, + is_optional_tool: false, + source_type: if false $(|| $external)* { + SourceType::Submodule + } else { + SourceType::InTree + }, + extra_features: vec![], + }).expect("expected to build -- essential tool") + } + } + )+ + } +} + +bootstrap_tool!( + Rustbook, "src/tools/rustbook", "rustbook"; + UnstableBookGen, "src/tools/unstable-book-gen", "unstable-book-gen"; + Tidy, "src/tools/tidy", "tidy"; + Linkchecker, "src/tools/linkchecker", "linkchecker"; + CargoTest, "src/tools/cargotest", "cargotest"; + Compiletest, "src/tools/compiletest", "compiletest", is_unstable_tool = true; + BuildManifest, "src/tools/build-manifest", "build-manifest"; + RemoteTestClient, "src/tools/remote-test-client", "remote-test-client"; + RustInstaller, "src/tools/rust-installer", "rust-installer", is_external_tool = true; + RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes"; + ExpandYamlAnchors, "src/tools/expand-yaml-anchors", "expand-yaml-anchors"; + LintDocs, "src/tools/lint-docs", "lint-docs"; + JsonDocCk, "src/tools/jsondocck", "jsondocck"; + HtmlChecker, "src/tools/html-checker", "html-checker"; + BumpStage0, "src/tools/bump-stage0", "bump-stage0"; +); + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] +pub struct ErrorIndex { + pub compiler: Compiler, +} + +impl ErrorIndex { + pub fn command(builder: &Builder<'_>) -> Command { + // Error-index-generator links with the rustdoc library, so we need to add `rustc_lib_paths` + // for rustc_private and libLLVM.so, and `sysroot_lib` for libstd, etc. + let host = builder.config.build; + let compiler = builder.compiler_for(builder.top_stage, host, host); + let mut cmd = Command::new(builder.ensure(ErrorIndex { compiler })); + let mut dylib_paths = builder.rustc_lib_paths(compiler); + dylib_paths.push(PathBuf::from(&builder.sysroot_libdir(compiler, compiler.host))); + add_dylib_path(dylib_paths, &mut cmd); + cmd + } +} + +impl Step for ErrorIndex { + type Output = PathBuf; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/error_index_generator") + } + + fn make_run(run: RunConfig<'_>) { + // Compile the error-index in the same stage as rustdoc to avoid + // recompiling rustdoc twice if we can. + // + // NOTE: This `make_run` isn't used in normal situations, only if you + // manually build the tool with `x.py build + // src/tools/error-index-generator` which almost nobody does. + // Normally, `x.py test` or `x.py doc` will use the + // `ErrorIndex::command` function instead. + let compiler = + run.builder.compiler(run.builder.top_stage.saturating_sub(1), run.builder.config.build); + run.builder.ensure(ErrorIndex { compiler }); + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + builder + .ensure(ToolBuild { + compiler: self.compiler, + target: self.compiler.host, + tool: "error_index_generator", + mode: Mode::ToolRustc, + path: "src/tools/error_index_generator", + is_optional_tool: false, + source_type: SourceType::InTree, + extra_features: Vec::new(), + }) + .expect("expected to build -- essential tool") + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RemoteTestServer { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for RemoteTestServer { + type Output = PathBuf; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/remote-test-server") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RemoteTestServer { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + builder + .ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "remote-test-server", + mode: Mode::ToolStd, + path: "src/tools/remote-test-server", + is_optional_tool: false, + source_type: SourceType::InTree, + extra_features: Vec::new(), + }) + .expect("expected to build -- essential tool") + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] +pub struct Rustdoc { + /// This should only ever be 0 or 2. + /// We sometimes want to reference the "bootstrap" rustdoc, which is why this option is here. + pub compiler: Compiler, +} + +impl Step for Rustdoc { + type Output = PathBuf; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rustdoc").path("src/librustdoc") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Rustdoc { + // Note: this is somewhat unique in that we actually want a *target* + // compiler here, because rustdoc *is* a compiler. We won't be using + // this as the compiler to build with, but rather this is "what + // compiler are we producing"? + compiler: run.builder.compiler(run.builder.top_stage, run.target), + }); + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + let target_compiler = self.compiler; + if target_compiler.stage == 0 { + if !target_compiler.is_snapshot(builder) { + panic!("rustdoc in stage 0 must be snapshot rustdoc"); + } + return builder.initial_rustc.with_file_name(exe("rustdoc", target_compiler.host)); + } + let target = target_compiler.host; + // Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise + // we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage + // compilers, which isn't what we want. Rustdoc should be linked in the same way as the + // rustc compiler it's paired with, so it must be built with the previous stage compiler. + let build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build); + + // When using `download-rustc` and a stage0 build_compiler, copying rustc doesn't actually + // build stage0 libstd (because the libstd in sysroot has the wrong ABI). Explicitly build + // it. + builder.ensure(compile::Std::new(build_compiler, target_compiler.host)); + builder.ensure(compile::Rustc::new(build_compiler, target_compiler.host)); + // NOTE: this implies that `download-rustc` is pretty useless when compiling with the stage0 + // compiler, since you do just as much work. + if !builder.config.dry_run && builder.download_rustc() && build_compiler.stage == 0 { + println!( + "warning: `download-rustc` does nothing when building stage1 tools; consider using `--stage 2` instead" + ); + } + + // The presence of `target_compiler` ensures that the necessary libraries (codegen backends, + // compiler libraries, ...) are built. Rustdoc does not require the presence of any + // libraries within sysroot_libdir (i.e., rustlib), though doctests may want it (since + // they'll be linked to those libraries). As such, don't explicitly `ensure` any additional + // libraries here. The intuition here is that If we've built a compiler, we should be able + // to build rustdoc. + // + let mut features = Vec::new(); + if builder.config.jemalloc { + features.push("jemalloc".to_string()); + } + + let cargo = prepare_tool_cargo( + builder, + build_compiler, + Mode::ToolRustc, + target, + "build", + "src/tools/rustdoc", + SourceType::InTree, + features.as_slice(), + ); + + builder.info(&format!( + "Building rustdoc for stage{} ({})", + target_compiler.stage, target_compiler.host + )); + builder.run(&mut cargo.into()); + + // Cargo adds a number of paths to the dylib search path on windows, which results in + // the wrong rustdoc being executed. To avoid the conflicting rustdocs, we name the "tool" + // rustdoc a different name. + let tool_rustdoc = builder + .cargo_out(build_compiler, Mode::ToolRustc, target) + .join(exe("rustdoc_tool_binary", target_compiler.host)); + + // don't create a stage0-sysroot/bin directory. + if target_compiler.stage > 0 { + let sysroot = builder.sysroot(target_compiler); + let bindir = sysroot.join("bin"); + t!(fs::create_dir_all(&bindir)); + let bin_rustdoc = bindir.join(exe("rustdoc", target_compiler.host)); + let _ = fs::remove_file(&bin_rustdoc); + builder.copy(&tool_rustdoc, &bin_rustdoc); + bin_rustdoc + } else { + tool_rustdoc + } + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Cargo { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Cargo { + type Output = PathBuf; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/tools/cargo").default_condition( + builder.config.extended + && builder.config.tools.as_ref().map_or( + true, + // If `tools` is set, search list for this tool. + |tools| tools.iter().any(|tool| tool == "cargo"), + ), + ) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Cargo { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + let cargo_bin_path = builder + .ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "cargo", + mode: Mode::ToolRustc, + path: "src/tools/cargo", + is_optional_tool: false, + source_type: SourceType::Submodule, + extra_features: Vec::new(), + }) + .expect("expected to build -- essential tool"); + + let build_cred = |name, path| { + // These credential helpers are currently experimental. + // Any build failures will be ignored. + let _ = builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: name, + mode: Mode::ToolRustc, + path, + is_optional_tool: true, + source_type: SourceType::Submodule, + extra_features: Vec::new(), + }); + }; + + if self.target.contains("windows") { + build_cred( + "cargo-credential-wincred", + "src/tools/cargo/crates/credential/cargo-credential-wincred", + ); + } + if self.target.contains("apple-darwin") { + build_cred( + "cargo-credential-macos-keychain", + "src/tools/cargo/crates/credential/cargo-credential-macos-keychain", + ); + } + build_cred( + "cargo-credential-1password", + "src/tools/cargo/crates/credential/cargo-credential-1password", + ); + cargo_bin_path + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct LldWrapper { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for LldWrapper { + type Output = PathBuf; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + let src_exe = builder + .ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "lld-wrapper", + mode: Mode::ToolStd, + path: "src/tools/lld-wrapper", + is_optional_tool: false, + source_type: SourceType::InTree, + extra_features: Vec::new(), + }) + .expect("expected to build -- essential tool"); + + src_exe + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustAnalyzer { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for RustAnalyzer { + type Output = Option<PathBuf>; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/tools/rust-analyzer").default_condition( + builder.config.extended + && builder + .config + .tools + .as_ref() + .map_or(true, |tools| tools.iter().any(|tool| tool == "rust-analyzer")), + ) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustAnalyzer { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option<PathBuf> { + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "rust-analyzer", + mode: Mode::ToolStd, + path: "src/tools/rust-analyzer", + extra_features: vec!["rust-analyzer/in-rust-tree".to_owned()], + is_optional_tool: false, + source_type: SourceType::InTree, + }) + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustAnalyzerProcMacroSrv { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for RustAnalyzerProcMacroSrv { + type Output = Option<PathBuf>; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/tools/rust-analyzer").default_condition( + builder.config.extended + && builder + .config + .tools + .as_ref() + .map_or(true, |tools| tools.iter().any(|tool| tool == "rust-analyzer")), + ) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustAnalyzerProcMacroSrv { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option<PathBuf> { + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "rust-analyzer-proc-macro-srv", + mode: Mode::ToolStd, + path: "src/tools/rust-analyzer/crates/proc-macro-srv-cli", + extra_features: vec!["proc-macro-srv/sysroot-abi".to_owned()], + is_optional_tool: false, + source_type: SourceType::InTree, + }) + } +} + +macro_rules! tool_extended { + (($sel:ident, $builder:ident), + $($name:ident, + $toolstate:ident, + $path:expr, + $tool_name:expr, + stable = $stable:expr, + $(in_tree = $in_tree:expr,)? + $(submodule = $submodule:literal,)? + $(tool_std = $tool_std:literal,)? + $extra_deps:block;)+) => { + $( + #[derive(Debug, Clone, Hash, PartialEq, Eq)] + pub struct $name { + pub compiler: Compiler, + pub target: TargetSelection, + pub extra_features: Vec<String>, + } + + impl Step for $name { + type Output = Option<PathBuf>; + const DEFAULT: bool = true; // Overwritten below + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path($path).default_condition( + builder.config.extended + && builder.config.tools.as_ref().map_or( + // By default, on nightly/dev enable all tools, else only + // build stable tools. + $stable || builder.build.unstable_features(), + // If `tools` is set, search list for this tool. + |tools| { + tools.iter().any(|tool| match tool.as_ref() { + "clippy" => $tool_name == "clippy-driver", + x => $tool_name == x, + }) + }), + ) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure($name { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + extra_features: Vec::new(), + }); + } + + #[allow(unused_mut)] + fn run(mut $sel, $builder: &Builder<'_>) -> Option<PathBuf> { + $extra_deps + $( $builder.update_submodule(&Path::new("src").join("tools").join($submodule)); )? + $builder.ensure(ToolBuild { + compiler: $sel.compiler, + target: $sel.target, + tool: $tool_name, + mode: if false $(|| $tool_std)? { Mode::ToolStd } else { Mode::ToolRustc }, + path: $path, + extra_features: $sel.extra_features, + is_optional_tool: true, + source_type: if false $(|| $in_tree)* { + SourceType::InTree + } else { + SourceType::Submodule + }, + }) + } + } + )+ + } +} + +// Note: tools need to be also added to `Builder::get_step_descriptions` in `builder.rs` +// to make `./x.py build <tool>` work. +// Note: Most submodule updates for tools are handled by bootstrap.py, since they're needed just to +// invoke Cargo to build bootstrap. See the comment there for more details. +tool_extended!((self, builder), + Cargofmt, rustfmt, "src/tools/rustfmt", "cargo-fmt", stable=true, in_tree=true, {}; + CargoClippy, clippy, "src/tools/clippy", "cargo-clippy", stable=true, in_tree=true, {}; + Clippy, clippy, "src/tools/clippy", "clippy-driver", stable=true, in_tree=true, {}; + Miri, miri, "src/tools/miri", "miri", stable=false, {}; + CargoMiri, miri, "src/tools/miri/cargo-miri", "cargo-miri", stable=false, {}; + Rls, rls, "src/tools/rls", "rls", stable=true, { + builder.ensure(Clippy { + compiler: self.compiler, + target: self.target, + extra_features: Vec::new(), + }); + self.extra_features.push("clippy".to_owned()); + }; + // FIXME: tool_std is not quite right, we shouldn't allow nightly features. + // But `builder.cargo` doesn't know how to handle ToolBootstrap in stages other than 0, + // and this is close enough for now. + RustDemangler, rust_demangler, "src/tools/rust-demangler", "rust-demangler", stable=false, in_tree=true, tool_std=true, {}; + Rustfmt, rustfmt, "src/tools/rustfmt", "rustfmt", stable=true, in_tree=true, {}; +); + +impl<'a> Builder<'a> { + /// Gets a `Command` which is ready to run `tool` in `stage` built for + /// `host`. + pub fn tool_cmd(&self, tool: Tool) -> Command { + let mut cmd = Command::new(self.tool_exe(tool)); + let compiler = self.compiler(0, self.config.build); + let host = &compiler.host; + // Prepares the `cmd` provided to be able to run the `compiler` provided. + // + // Notably this munges the dynamic library lookup path to point to the + // right location to run `compiler`. + let mut lib_paths: Vec<PathBuf> = vec![ + self.build.rustc_snapshot_libdir(), + self.cargo_out(compiler, Mode::ToolBootstrap, *host).join("deps"), + ]; + + // On MSVC a tool may invoke a C compiler (e.g., compiletest in run-make + // mode) and that C compiler may need some extra PATH modification. Do + // so here. + if compiler.host.contains("msvc") { + let curpaths = env::var_os("PATH").unwrap_or_default(); + let curpaths = env::split_paths(&curpaths).collect::<Vec<_>>(); + for &(ref k, ref v) in self.cc[&compiler.host].env() { + if k != "PATH" { + continue; + } + for path in env::split_paths(v) { + if !curpaths.contains(&path) { + lib_paths.push(path); + } + } + } + } + + add_dylib_path(lib_paths, &mut cmd); + + // Provide a RUSTC for this command to use. + cmd.env("RUSTC", &self.initial_rustc); + + cmd + } +} diff --git a/src/bootstrap/toolstate.rs b/src/bootstrap/toolstate.rs new file mode 100644 index 000000000..2cfeae7dc --- /dev/null +++ b/src/bootstrap/toolstate.rs @@ -0,0 +1,479 @@ +use crate::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::util::t; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::env; +use std::fmt; +use std::fs; +use std::io::{Seek, SeekFrom}; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time; + +// Each cycle is 42 days long (6 weeks); the last week is 35..=42 then. +const BETA_WEEK_START: u64 = 35; + +#[cfg(target_os = "linux")] +const OS: Option<&str> = Some("linux"); + +#[cfg(windows)] +const OS: Option<&str> = Some("windows"); + +#[cfg(all(not(target_os = "linux"), not(windows)))] +const OS: Option<&str> = None; + +type ToolstateData = HashMap<Box<str>, ToolState>; + +#[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, PartialOrd)] +#[serde(rename_all = "kebab-case")] +/// Whether a tool can be compiled, tested or neither +pub enum ToolState { + /// The tool compiles successfully, but the test suite fails + TestFail = 1, + /// The tool compiles successfully and its test suite passes + TestPass = 2, + /// The tool can't even be compiled + BuildFail = 0, +} + +impl fmt::Display for ToolState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}", + match self { + ToolState::TestFail => "test-fail", + ToolState::TestPass => "test-pass", + ToolState::BuildFail => "build-fail", + } + ) + } +} + +/// Number of days after the last promotion of beta. +/// Its value is 41 on the Tuesday where "Promote master to beta (T-2)" happens. +/// The Wednesday after this has value 0. +/// We track this value to prevent regressing tools in the last week of the 6-week cycle. +fn days_since_beta_promotion() -> u64 { + let since_epoch = t!(time::SystemTime::UNIX_EPOCH.elapsed()); + (since_epoch.as_secs() / 86400 - 20) % 42 +} + +// These tools must test-pass on the beta/stable channels. +// +// On the nightly channel, their build step must be attempted, but they may not +// be able to build successfully. +static STABLE_TOOLS: &[(&str, &str)] = &[ + ("book", "src/doc/book"), + ("nomicon", "src/doc/nomicon"), + ("reference", "src/doc/reference"), + ("rust-by-example", "src/doc/rust-by-example"), + ("edition-guide", "src/doc/edition-guide"), + ("rls", "src/tools/rls"), +]; + +// These tools are permitted to not build on the beta/stable channels. +// +// We do require that we checked whether they build or not on the tools builder, +// though, as otherwise we will be unable to file an issue if they start +// failing. +static NIGHTLY_TOOLS: &[(&str, &str)] = &[ + ("miri", "src/tools/miri"), + ("embedded-book", "src/doc/embedded-book"), + // ("rustc-dev-guide", "src/doc/rustc-dev-guide"), +]; + +fn print_error(tool: &str, submodule: &str) { + eprintln!(); + eprintln!("We detected that this PR updated '{}', but its tests failed.", tool); + eprintln!(); + eprintln!("If you do intend to update '{}', please check the error messages above and", tool); + eprintln!("commit another update."); + eprintln!(); + eprintln!("If you do NOT intend to update '{}', please ensure you did not accidentally", tool); + eprintln!("change the submodule at '{}'. You may ask your reviewer for the", submodule); + eprintln!("proper steps."); + crate::detail_exit(3); +} + +fn check_changed_files(toolstates: &HashMap<Box<str>, ToolState>) { + // Changed files + let output = std::process::Command::new("git") + .arg("diff") + .arg("--name-status") + .arg("HEAD") + .arg("HEAD^") + .output(); + let output = match output { + Ok(o) => o, + Err(e) => { + eprintln!("Failed to get changed files: {:?}", e); + crate::detail_exit(1); + } + }; + + let output = t!(String::from_utf8(output.stdout)); + + for (tool, submodule) in STABLE_TOOLS.iter().chain(NIGHTLY_TOOLS.iter()) { + let changed = output.lines().any(|l| l.starts_with('M') && l.ends_with(submodule)); + eprintln!("Verifying status of {}...", tool); + if !changed { + continue; + } + + eprintln!("This PR updated '{}', verifying if status is 'test-pass'...", submodule); + if toolstates[*tool] != ToolState::TestPass { + print_error(tool, submodule); + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct ToolStateCheck; + +impl Step for ToolStateCheck { + type Output = (); + + /// Checks tool state status. + /// + /// This is intended to be used in the `checktools.sh` script. To use + /// this, set `save-toolstates` in `config.toml` so that tool status will + /// be saved to a JSON file. Then, run `x.py test --no-fail-fast` for all + /// of the tools to populate the JSON file. After that is done, this + /// command can be run to check for any status failures, and exits with an + /// error if there are any. + /// + /// This also handles publishing the results to the `history` directory of + /// the toolstate repo <https://github.com/rust-lang-nursery/rust-toolstate> + /// if the env var `TOOLSTATE_PUBLISH` is set. Note that there is a + /// *separate* step of updating the `latest.json` file and creating GitHub + /// issues and comments in `src/ci/publish_toolstate.sh`, which is only + /// performed on master. (The shell/python code is intended to be migrated + /// here eventually.) + /// + /// The rules for failure are: + /// * If the PR modifies a tool, the status must be test-pass. + /// NOTE: There is intent to change this, see + /// <https://github.com/rust-lang/rust/issues/65000>. + /// * All "stable" tools must be test-pass on the stable or beta branches. + /// * During beta promotion week, a PR is not allowed to "regress" a + /// stable tool. That is, the status is not allowed to get worse + /// (test-pass to test-fail or build-fail). + fn run(self, builder: &Builder<'_>) { + if builder.config.dry_run { + return; + } + + let days_since_beta_promotion = days_since_beta_promotion(); + let in_beta_week = days_since_beta_promotion >= BETA_WEEK_START; + let is_nightly = !(builder.config.channel == "beta" || builder.config.channel == "stable"); + let toolstates = builder.toolstates(); + + let mut did_error = false; + + for (tool, _) in STABLE_TOOLS.iter().chain(NIGHTLY_TOOLS.iter()) { + if !toolstates.contains_key(*tool) { + did_error = true; + eprintln!("error: Tool `{}` was not recorded in tool state.", tool); + } + } + + if did_error { + crate::detail_exit(1); + } + + check_changed_files(&toolstates); + checkout_toolstate_repo(); + let old_toolstate = read_old_toolstate(); + + for (tool, _) in STABLE_TOOLS.iter() { + let state = toolstates[*tool]; + + if state != ToolState::TestPass { + if !is_nightly { + did_error = true; + eprintln!("error: Tool `{}` should be test-pass but is {}", tool, state); + } else if in_beta_week { + let old_state = old_toolstate + .iter() + .find(|ts| ts.tool == *tool) + .expect("latest.json missing tool") + .state(); + if state < old_state { + did_error = true; + eprintln!( + "error: Tool `{}` has regressed from {} to {} during beta week.", + tool, old_state, state + ); + } else { + // This warning only appears in the logs, which most + // people won't read. It's mostly here for testing and + // debugging. + eprintln!( + "warning: Tool `{}` is not test-pass (is `{}`), \ + this should be fixed before beta is branched.", + tool, state + ); + } + } + // `publish_toolstate.py` is responsible for updating + // `latest.json` and creating comments/issues warning people + // if there is a regression. That all happens in a separate CI + // job on the master branch once the PR has passed all tests + // on the `auto` branch. + } + } + + if did_error { + crate::detail_exit(1); + } + + if builder.config.channel == "nightly" && env::var_os("TOOLSTATE_PUBLISH").is_some() { + commit_toolstate_change(&toolstates); + } + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("check-tools") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(ToolStateCheck); + } +} + +impl Builder<'_> { + fn toolstates(&self) -> HashMap<Box<str>, ToolState> { + if let Some(ref path) = self.config.save_toolstates { + if let Some(parent) = path.parent() { + // Ensure the parent directory always exists + t!(std::fs::create_dir_all(parent)); + } + let mut file = + t!(fs::OpenOptions::new().create(true).write(true).read(true).open(path)); + + serde_json::from_reader(&mut file).unwrap_or_default() + } else { + Default::default() + } + } + + /// Updates the actual toolstate of a tool. + /// + /// The toolstates are saved to the file specified by the key + /// `rust.save-toolstates` in `config.toml`. If unspecified, nothing will be + /// done. The file is updated immediately after this function completes. + pub fn save_toolstate(&self, tool: &str, state: ToolState) { + // If we're in a dry run setting we don't want to save toolstates as + // that means if we e.g. panic down the line it'll look like we tested + // everything (but we actually haven't). + if self.config.dry_run { + return; + } + // Toolstate isn't tracked for clippy or rustfmt, but since most tools do, we avoid checking + // in all the places we could save toolstate and just do so here. + if tool == "clippy-driver" || tool == "rustfmt" { + return; + } + if let Some(ref path) = self.config.save_toolstates { + if let Some(parent) = path.parent() { + // Ensure the parent directory always exists + t!(std::fs::create_dir_all(parent)); + } + let mut file = + t!(fs::OpenOptions::new().create(true).read(true).write(true).open(path)); + + let mut current_toolstates: HashMap<Box<str>, ToolState> = + serde_json::from_reader(&mut file).unwrap_or_default(); + current_toolstates.insert(tool.into(), state); + t!(file.seek(SeekFrom::Start(0))); + t!(file.set_len(0)); + t!(serde_json::to_writer(file, ¤t_toolstates)); + } + } +} + +fn toolstate_repo() -> String { + env::var("TOOLSTATE_REPO") + .unwrap_or_else(|_| "https://github.com/rust-lang-nursery/rust-toolstate.git".to_string()) +} + +/// Directory where the toolstate repo is checked out. +const TOOLSTATE_DIR: &str = "rust-toolstate"; + +/// Checks out the toolstate repo into `TOOLSTATE_DIR`. +fn checkout_toolstate_repo() { + if let Ok(token) = env::var("TOOLSTATE_REPO_ACCESS_TOKEN") { + prepare_toolstate_config(&token); + } + if Path::new(TOOLSTATE_DIR).exists() { + eprintln!("Cleaning old toolstate directory..."); + t!(fs::remove_dir_all(TOOLSTATE_DIR)); + } + + let status = Command::new("git") + .arg("clone") + .arg("--depth=1") + .arg(toolstate_repo()) + .arg(TOOLSTATE_DIR) + .status(); + let success = match status { + Ok(s) => s.success(), + Err(_) => false, + }; + if !success { + panic!("git clone unsuccessful (status: {:?})", status); + } +} + +/// Sets up config and authentication for modifying the toolstate repo. +fn prepare_toolstate_config(token: &str) { + fn git_config(key: &str, value: &str) { + let status = Command::new("git").arg("config").arg("--global").arg(key).arg(value).status(); + let success = match status { + Ok(s) => s.success(), + Err(_) => false, + }; + if !success { + panic!("git config key={} value={} failed (status: {:?})", key, value, status); + } + } + + // If changing anything here, then please check that `src/ci/publish_toolstate.sh` is up to date + // as well. + git_config("user.email", "7378925+rust-toolstate-update@users.noreply.github.com"); + git_config("user.name", "Rust Toolstate Update"); + git_config("credential.helper", "store"); + + let credential = format!("https://{}:x-oauth-basic@github.com\n", token,); + let git_credential_path = PathBuf::from(t!(env::var("HOME"))).join(".git-credentials"); + t!(fs::write(&git_credential_path, credential)); +} + +/// Reads the latest toolstate from the toolstate repo. +fn read_old_toolstate() -> Vec<RepoState> { + let latest_path = Path::new(TOOLSTATE_DIR).join("_data").join("latest.json"); + let old_toolstate = t!(fs::read(latest_path)); + t!(serde_json::from_slice(&old_toolstate)) +} + +/// This function `commit_toolstate_change` provides functionality for pushing a change +/// to the `rust-toolstate` repository. +/// +/// The function relies on a GitHub bot user, which should have a Personal access +/// token defined in the environment variable $TOOLSTATE_REPO_ACCESS_TOKEN. If for +/// some reason you need to change the token, please update the Azure Pipelines +/// variable group. +/// +/// 1. Generate a new Personal access token: +/// +/// * Login to the bot account, and go to Settings -> Developer settings -> +/// Personal access tokens +/// * Click "Generate new token" +/// * Enable the "public_repo" permission, then click "Generate token" +/// * Copy the generated token (should be a 40-digit hexadecimal number). +/// Save it somewhere secure, as the token would be gone once you leave +/// the page. +/// +/// 2. Update the variable group in Azure Pipelines +/// +/// * Ping a member of the infrastructure team to do this. +/// +/// 4. Replace the email address below if the bot account identity is changed +/// +/// * See <https://help.github.com/articles/about-commit-email-addresses/> +/// if a private email by GitHub is wanted. +fn commit_toolstate_change(current_toolstate: &ToolstateData) { + let message = format!("({} CI update)", OS.expect("linux/windows only")); + let mut success = false; + for _ in 1..=5 { + // Upload the test results (the new commit-to-toolstate mapping) to the toolstate repo. + // This does *not* change the "current toolstate"; that only happens post-landing + // via `src/ci/docker/publish_toolstate.sh`. + publish_test_results(¤t_toolstate); + + // `git commit` failing means nothing to commit. + let status = t!(Command::new("git") + .current_dir(TOOLSTATE_DIR) + .arg("commit") + .arg("-a") + .arg("-m") + .arg(&message) + .status()); + if !status.success() { + success = true; + break; + } + + let status = t!(Command::new("git") + .current_dir(TOOLSTATE_DIR) + .arg("push") + .arg("origin") + .arg("master") + .status()); + // If we successfully push, exit. + if status.success() { + success = true; + break; + } + eprintln!("Sleeping for 3 seconds before retrying push"); + std::thread::sleep(std::time::Duration::from_secs(3)); + let status = t!(Command::new("git") + .current_dir(TOOLSTATE_DIR) + .arg("fetch") + .arg("origin") + .arg("master") + .status()); + assert!(status.success()); + let status = t!(Command::new("git") + .current_dir(TOOLSTATE_DIR) + .arg("reset") + .arg("--hard") + .arg("origin/master") + .status()); + assert!(status.success()); + } + + if !success { + panic!("Failed to update toolstate repository with new data"); + } +} + +/// Updates the "history" files with the latest results. +/// +/// These results will later be promoted to `latest.json` by the +/// `publish_toolstate.py` script if the PR passes all tests and is merged to +/// master. +fn publish_test_results(current_toolstate: &ToolstateData) { + let commit = t!(std::process::Command::new("git").arg("rev-parse").arg("HEAD").output()); + let commit = t!(String::from_utf8(commit.stdout)); + + let toolstate_serialized = t!(serde_json::to_string(¤t_toolstate)); + + let history_path = Path::new(TOOLSTATE_DIR) + .join("history") + .join(format!("{}.tsv", OS.expect("linux/windows only"))); + let mut file = t!(fs::read_to_string(&history_path)); + let end_of_first_line = file.find('\n').unwrap(); + file.insert_str(end_of_first_line, &format!("\n{}\t{}", commit.trim(), toolstate_serialized)); + t!(fs::write(&history_path, file)); +} + +#[derive(Debug, Deserialize)] +struct RepoState { + tool: String, + windows: ToolState, + linux: ToolState, +} + +impl RepoState { + fn state(&self) -> ToolState { + if cfg!(target_os = "linux") { + self.linux + } else if cfg!(windows) { + self.windows + } else { + unimplemented!() + } + } +} diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs new file mode 100644 index 000000000..1895e2901 --- /dev/null +++ b/src/bootstrap/util.rs @@ -0,0 +1,603 @@ +//! Various utility functions used throughout rustbuild. +//! +//! Simple things like testing the various filesystem operations here and there, +//! not a lot of interesting happenings here unfortunately. + +use std::env; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::str; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +use crate::builder::Builder; +use crate::config::{Config, TargetSelection}; + +/// A helper macro to `unwrap` a result except also print out details like: +/// +/// * The file/line of the panic +/// * The expression that failed +/// * The error itself +/// +/// This is currently used judiciously throughout the build system rather than +/// using a `Result` with `try!`, but this may change one day... +#[macro_export] +macro_rules! t { + ($e:expr) => { + match $e { + Ok(e) => e, + Err(e) => panic!("{} failed with {}", stringify!($e), e), + } + }; + // it can show extra info in the second parameter + ($e:expr, $extra:expr) => { + match $e { + Ok(e) => e, + Err(e) => panic!("{} failed with {} ({:?})", stringify!($e), e, $extra), + } + }; +} +pub use t; + +/// Given an executable called `name`, return the filename for the +/// executable for a particular target. +pub fn exe(name: &str, target: TargetSelection) -> String { + if target.contains("windows") { format!("{}.exe", name) } else { name.to_string() } +} + +/// Returns `true` if the file name given looks like a dynamic library. +pub fn is_dylib(name: &str) -> bool { + name.ends_with(".dylib") || name.ends_with(".so") || name.ends_with(".dll") +} + +/// Returns `true` if the file name given looks like a debug info file +pub fn is_debug_info(name: &str) -> bool { + // FIXME: consider split debug info on other platforms (e.g., Linux, macOS) + name.ends_with(".pdb") +} + +/// Returns the corresponding relative library directory that the compiler's +/// dylibs will be found in. +pub fn libdir(target: TargetSelection) -> &'static str { + if target.contains("windows") { "bin" } else { "lib" } +} + +/// Adds a list of lookup paths to `cmd`'s dynamic library lookup path. +/// If the dylib_path_var is already set for this cmd, the old value will be overwritten! +pub fn add_dylib_path(path: Vec<PathBuf>, cmd: &mut Command) { + let mut list = dylib_path(); + for path in path { + list.insert(0, path); + } + cmd.env(dylib_path_var(), t!(env::join_paths(list))); +} + +include!("dylib_util.rs"); + +/// Adds a list of lookup paths to `cmd`'s link library lookup path. +pub fn add_link_lib_path(path: Vec<PathBuf>, cmd: &mut Command) { + let mut list = link_lib_path(); + for path in path { + list.insert(0, path); + } + cmd.env(link_lib_path_var(), t!(env::join_paths(list))); +} + +/// Returns the environment variable which the link library lookup path +/// resides in for this platform. +fn link_lib_path_var() -> &'static str { + if cfg!(target_env = "msvc") { "LIB" } else { "LIBRARY_PATH" } +} + +/// Parses the `link_lib_path_var()` environment variable, returning a list of +/// paths that are members of this lookup path. +fn link_lib_path() -> Vec<PathBuf> { + let var = match env::var_os(link_lib_path_var()) { + Some(v) => v, + None => return vec![], + }; + env::split_paths(&var).collect() +} + +pub struct TimeIt(bool, Instant); + +/// Returns an RAII structure that prints out how long it took to drop. +pub fn timeit(builder: &Builder<'_>) -> TimeIt { + TimeIt(builder.config.dry_run, Instant::now()) +} + +impl Drop for TimeIt { + fn drop(&mut self) { + let time = self.1.elapsed(); + if !self.0 { + println!("\tfinished in {}.{:03} seconds", time.as_secs(), time.subsec_millis()); + } + } +} + +/// Used for download caching +pub(crate) fn program_out_of_date(stamp: &Path, key: &str) -> bool { + if !stamp.exists() { + return true; + } + t!(fs::read_to_string(stamp)) != key +} + +/// Symlinks two directories, using junctions on Windows and normal symlinks on +/// Unix. +pub fn symlink_dir(config: &Config, src: &Path, dest: &Path) -> io::Result<()> { + if config.dry_run { + return Ok(()); + } + let _ = fs::remove_dir(dest); + return symlink_dir_inner(src, dest); + + #[cfg(not(windows))] + fn symlink_dir_inner(src: &Path, dest: &Path) -> io::Result<()> { + use std::os::unix::fs; + fs::symlink(src, dest) + } + + // Creating a directory junction on windows involves dealing with reparse + // points and the DeviceIoControl function, and this code is a skeleton of + // what can be found here: + // + // http://www.flexhex.com/docs/articles/hard-links.phtml + #[cfg(windows)] + fn symlink_dir_inner(target: &Path, junction: &Path) -> io::Result<()> { + use std::ffi::OsStr; + use std::os::windows::ffi::OsStrExt; + use std::ptr; + + use winapi::shared::minwindef::{DWORD, WORD}; + use winapi::um::fileapi::{CreateFileW, OPEN_EXISTING}; + use winapi::um::handleapi::CloseHandle; + use winapi::um::ioapiset::DeviceIoControl; + use winapi::um::winbase::{FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT}; + use winapi::um::winioctl::FSCTL_SET_REPARSE_POINT; + use winapi::um::winnt::{ + FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, GENERIC_WRITE, + IO_REPARSE_TAG_MOUNT_POINT, MAXIMUM_REPARSE_DATA_BUFFER_SIZE, WCHAR, + }; + + #[allow(non_snake_case)] + #[repr(C)] + struct REPARSE_MOUNTPOINT_DATA_BUFFER { + ReparseTag: DWORD, + ReparseDataLength: DWORD, + Reserved: WORD, + ReparseTargetLength: WORD, + ReparseTargetMaximumLength: WORD, + Reserved1: WORD, + ReparseTarget: WCHAR, + } + + fn to_u16s<S: AsRef<OsStr>>(s: S) -> io::Result<Vec<u16>> { + Ok(s.as_ref().encode_wide().chain(Some(0)).collect()) + } + + // We're using low-level APIs to create the junction, and these are more + // picky about paths. For example, forward slashes cannot be used as a + // path separator, so we should try to canonicalize the path first. + let target = fs::canonicalize(target)?; + + fs::create_dir(junction)?; + + let path = to_u16s(junction)?; + + unsafe { + let h = CreateFileW( + path.as_ptr(), + GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + ptr::null_mut(), + OPEN_EXISTING, + FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS, + ptr::null_mut(), + ); + + let mut data = [0u8; MAXIMUM_REPARSE_DATA_BUFFER_SIZE as usize]; + let db = data.as_mut_ptr() as *mut REPARSE_MOUNTPOINT_DATA_BUFFER; + let buf = &mut (*db).ReparseTarget as *mut u16; + let mut i = 0; + // FIXME: this conversion is very hacky + let v = br"\??\"; + let v = v.iter().map(|x| *x as u16); + for c in v.chain(target.as_os_str().encode_wide().skip(4)) { + *buf.offset(i) = c; + i += 1; + } + *buf.offset(i) = 0; + i += 1; + (*db).ReparseTag = IO_REPARSE_TAG_MOUNT_POINT; + (*db).ReparseTargetMaximumLength = (i * 2) as WORD; + (*db).ReparseTargetLength = ((i - 1) * 2) as WORD; + (*db).ReparseDataLength = (*db).ReparseTargetLength as DWORD + 12; + + let mut ret = 0; + let res = DeviceIoControl( + h as *mut _, + FSCTL_SET_REPARSE_POINT, + data.as_ptr() as *mut _, + (*db).ReparseDataLength + 8, + ptr::null_mut(), + 0, + &mut ret, + ptr::null_mut(), + ); + + let out = if res == 0 { Err(io::Error::last_os_error()) } else { Ok(()) }; + CloseHandle(h); + out + } + } +} + +/// The CI environment rustbuild is running in. This mainly affects how the logs +/// are printed. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub enum CiEnv { + /// Not a CI environment. + None, + /// The Azure Pipelines environment, for Linux (including Docker), Windows, and macOS builds. + AzurePipelines, + /// The GitHub Actions environment, for Linux (including Docker), Windows and macOS builds. + GitHubActions, +} + +impl CiEnv { + /// Obtains the current CI environment. + pub fn current() -> CiEnv { + if env::var("TF_BUILD").map_or(false, |e| e == "True") { + CiEnv::AzurePipelines + } else if env::var("GITHUB_ACTIONS").map_or(false, |e| e == "true") { + CiEnv::GitHubActions + } else { + CiEnv::None + } + } + + /// If in a CI environment, forces the command to run with colors. + pub fn force_coloring_in_ci(self, cmd: &mut Command) { + if self != CiEnv::None { + // Due to use of stamp/docker, the output stream of rustbuild is not + // a TTY in CI, so coloring is by-default turned off. + // The explicit `TERM=xterm` environment is needed for + // `--color always` to actually work. This env var was lost when + // compiling through the Makefile. Very strange. + cmd.env("TERM", "xterm").args(&["--color", "always"]); + } + } +} + +pub fn forcing_clang_based_tests() -> bool { + if let Some(var) = env::var_os("RUSTBUILD_FORCE_CLANG_BASED_TESTS") { + match &var.to_string_lossy().to_lowercase()[..] { + "1" | "yes" | "on" => true, + "0" | "no" | "off" => false, + other => { + // Let's make sure typos don't go unnoticed + panic!( + "Unrecognized option '{}' set in \ + RUSTBUILD_FORCE_CLANG_BASED_TESTS", + other + ) + } + } + } else { + false + } +} + +pub fn use_host_linker(target: TargetSelection) -> bool { + // FIXME: this information should be gotten by checking the linker flavor + // of the rustc target + !(target.contains("emscripten") + || target.contains("wasm32") + || target.contains("nvptx") + || target.contains("fortanix") + || target.contains("fuchsia") + || target.contains("bpf") + || target.contains("switch")) +} + +pub fn is_valid_test_suite_arg<'a, P: AsRef<Path>>( + path: &'a Path, + suite_path: P, + builder: &Builder<'_>, +) -> Option<&'a str> { + let suite_path = suite_path.as_ref(); + let path = match path.strip_prefix(".") { + Ok(p) => p, + Err(_) => path, + }; + if !path.starts_with(suite_path) { + return None; + } + let abs_path = builder.src.join(path); + let exists = abs_path.is_dir() || abs_path.is_file(); + if !exists { + panic!( + "Invalid test suite filter \"{}\": file or directory does not exist", + abs_path.display() + ); + } + // Since test suite paths are themselves directories, if we don't + // specify a directory or file, we'll get an empty string here + // (the result of the test suite directory without its suite prefix). + // Therefore, we need to filter these out, as only the first --test-args + // flag is respected, so providing an empty --test-args conflicts with + // any following it. + match path.strip_prefix(suite_path).ok().and_then(|p| p.to_str()) { + Some(s) if !s.is_empty() => Some(s), + _ => None, + } +} + +pub fn run(cmd: &mut Command, print_cmd_on_fail: bool) { + if !try_run(cmd, print_cmd_on_fail) { + crate::detail_exit(1); + } +} + +pub fn try_run(cmd: &mut Command, print_cmd_on_fail: bool) -> bool { + let status = match cmd.status() { + Ok(status) => status, + Err(e) => fail(&format!("failed to execute command: {:?}\nerror: {}", cmd, e)), + }; + if !status.success() && print_cmd_on_fail { + println!( + "\n\ncommand did not execute successfully: {:?}\n\ + expected success, got: {}\n\n", + cmd, status + ); + } + status.success() +} + +pub fn check_run(cmd: &mut Command, print_cmd_on_fail: bool) -> bool { + let status = match cmd.status() { + Ok(status) => status, + Err(e) => { + println!("failed to execute command: {:?}\nerror: {}", cmd, e); + return false; + } + }; + if !status.success() && print_cmd_on_fail { + println!( + "\n\ncommand did not execute successfully: {:?}\n\ + expected success, got: {}\n\n", + cmd, status + ); + } + status.success() +} + +pub fn run_suppressed(cmd: &mut Command) { + if !try_run_suppressed(cmd) { + crate::detail_exit(1); + } +} + +pub fn try_run_suppressed(cmd: &mut Command) -> bool { + let output = match cmd.output() { + Ok(status) => status, + Err(e) => fail(&format!("failed to execute command: {:?}\nerror: {}", cmd, e)), + }; + if !output.status.success() { + println!( + "\n\ncommand did not execute successfully: {:?}\n\ + expected success, got: {}\n\n\ + stdout ----\n{}\n\ + stderr ----\n{}\n\n", + cmd, + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + output.status.success() +} + +pub fn make(host: &str) -> PathBuf { + if host.contains("dragonfly") + || host.contains("freebsd") + || host.contains("netbsd") + || host.contains("openbsd") + { + PathBuf::from("gmake") + } else { + PathBuf::from("make") + } +} + +#[track_caller] +pub fn output(cmd: &mut Command) -> String { + let output = match cmd.stderr(Stdio::inherit()).output() { + Ok(status) => status, + Err(e) => fail(&format!("failed to execute command: {:?}\nerror: {}", cmd, e)), + }; + if !output.status.success() { + panic!( + "command did not execute successfully: {:?}\n\ + expected success, got: {}", + cmd, output.status + ); + } + String::from_utf8(output.stdout).unwrap() +} + +/// Returns the last-modified time for `path`, or zero if it doesn't exist. +pub fn mtime(path: &Path) -> SystemTime { + fs::metadata(path).and_then(|f| f.modified()).unwrap_or(UNIX_EPOCH) +} + +/// Returns `true` if `dst` is up to date given that the file or files in `src` +/// are used to generate it. +/// +/// Uses last-modified time checks to verify this. +pub fn up_to_date(src: &Path, dst: &Path) -> bool { + if !dst.exists() { + return false; + } + let threshold = mtime(dst); + let meta = match fs::metadata(src) { + Ok(meta) => meta, + Err(e) => panic!("source {:?} failed to get metadata: {}", src, e), + }; + if meta.is_dir() { + dir_up_to_date(src, threshold) + } else { + meta.modified().unwrap_or(UNIX_EPOCH) <= threshold + } +} + +fn dir_up_to_date(src: &Path, threshold: SystemTime) -> bool { + t!(fs::read_dir(src)).map(|e| t!(e)).all(|e| { + let meta = t!(e.metadata()); + if meta.is_dir() { + dir_up_to_date(&e.path(), threshold) + } else { + meta.modified().unwrap_or(UNIX_EPOCH) < threshold + } + }) +} + +fn fail(s: &str) -> ! { + eprintln!("\n\n{}\n\n", s); + crate::detail_exit(1); +} + +/// Copied from `std::path::absolute` until it stabilizes. +/// +/// FIXME: this shouldn't exist. +pub(crate) fn absolute(path: &Path) -> PathBuf { + if path.as_os_str().is_empty() { + panic!("can't make empty path absolute"); + } + #[cfg(unix)] + { + t!(absolute_unix(path), format!("could not make path absolute: {}", path.display())) + } + #[cfg(windows)] + { + t!(absolute_windows(path), format!("could not make path absolute: {}", path.display())) + } + #[cfg(not(any(unix, windows)))] + { + println!("warning: bootstrap is not supported on non-unix platforms"); + t!(std::fs::canonicalize(t!(std::env::current_dir()))).join(path) + } +} + +#[cfg(unix)] +/// Make a POSIX path absolute without changing its semantics. +fn absolute_unix(path: &Path) -> io::Result<PathBuf> { + // This is mostly a wrapper around collecting `Path::components`, with + // exceptions made where this conflicts with the POSIX specification. + // See 4.13 Pathname Resolution, IEEE Std 1003.1-2017 + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 + + use std::os::unix::prelude::OsStrExt; + let mut components = path.components(); + let path_os = path.as_os_str().as_bytes(); + + let mut normalized = if path.is_absolute() { + // "If a pathname begins with two successive <slash> characters, the + // first component following the leading <slash> characters may be + // interpreted in an implementation-defined manner, although more than + // two leading <slash> characters shall be treated as a single <slash> + // character." + if path_os.starts_with(b"//") && !path_os.starts_with(b"///") { + components.next(); + PathBuf::from("//") + } else { + PathBuf::new() + } + } else { + env::current_dir()? + }; + normalized.extend(components); + + // "Interfaces using pathname resolution may specify additional constraints + // when a pathname that does not name an existing directory contains at + // least one non- <slash> character and contains one or more trailing + // <slash> characters". + // A trailing <slash> is also meaningful if "a symbolic link is + // encountered during pathname resolution". + + if path_os.ends_with(b"/") { + normalized.push(""); + } + + Ok(normalized) +} + +#[cfg(windows)] +fn absolute_windows(path: &std::path::Path) -> std::io::Result<std::path::PathBuf> { + use std::ffi::OsString; + use std::io::Error; + use std::os::windows::ffi::{OsStrExt, OsStringExt}; + use std::ptr::null_mut; + #[link(name = "kernel32")] + extern "system" { + fn GetFullPathNameW( + lpFileName: *const u16, + nBufferLength: u32, + lpBuffer: *mut u16, + lpFilePart: *mut *const u16, + ) -> u32; + } + + unsafe { + // encode the path as UTF-16 + let path: Vec<u16> = path.as_os_str().encode_wide().chain([0]).collect(); + let mut buffer = Vec::new(); + // Loop until either success or failure. + loop { + // Try to get the absolute path + let len = GetFullPathNameW( + path.as_ptr(), + buffer.len().try_into().unwrap(), + buffer.as_mut_ptr(), + null_mut(), + ); + match len as usize { + // Failure + 0 => return Err(Error::last_os_error()), + // Buffer is too small, resize. + len if len > buffer.len() => buffer.resize(len, 0), + // Success! + len => { + buffer.truncate(len); + return Ok(OsString::from_wide(&buffer).into()); + } + } + } + } +} + +/// Adapted from https://github.com/llvm/llvm-project/blob/782e91224601e461c019e0a4573bbccc6094fbcd/llvm/cmake/modules/HandleLLVMOptions.cmake#L1058-L1079 +/// +/// When `clang-cl` is used with instrumentation, we need to add clang's runtime library resource +/// directory to the linker flags, otherwise there will be linker errors about the profiler runtime +/// missing. This function returns the path to that directory. +pub fn get_clang_cl_resource_dir(clang_cl_path: &str) -> PathBuf { + // Similar to how LLVM does it, to find clang's library runtime directory: + // - we ask `clang-cl` to locate the `clang_rt.builtins` lib. + let mut builtins_locator = Command::new(clang_cl_path); + builtins_locator.args(&["/clang:-print-libgcc-file-name", "/clang:--rtlib=compiler-rt"]); + + let clang_rt_builtins = output(&mut builtins_locator); + let clang_rt_builtins = Path::new(clang_rt_builtins.trim()); + assert!( + clang_rt_builtins.exists(), + "`clang-cl` must correctly locate the library runtime directory" + ); + + // - the profiler runtime will be located in the same directory as the builtins lib, like + // `$LLVM_DISTRO_ROOT/lib/clang/$LLVM_VERSION/lib/windows`. + let clang_rt_dir = clang_rt_builtins.parent().expect("The clang lib folder should exist"); + clang_rt_dir.to_path_buf() +} |